Merge pull request #12 from surveyjs/bug/10-signature

Signature isn't copied from the scanned document fix #10

Author: andrewtelnov

SPEC.md

@@ -90,6 +90,7 @@ console.log(result.data);               // Structured responses matching form sc
   - Matrix column keys: column `title`/`text` -> column `name` (or `value` when `name` is absent)
   - Matrix row keys: row `text` -> row `value`
   - Choice values for `radiogroup`, `dropdown`, `checkbox`, `tagbox`, `ranking`, `imagepicker`: display `text` -> canonical `value`
+  - `signaturepad` fields: captured signature image -> Base64 string value
 - **JSON Schema Adapter**: Support for standard JSON Schema.
 - **Custom Adapter**: Simple interface for users to define their own mapping.
 

prompts/milestone-3-adapter-layer.md

@@ -33,6 +33,7 @@ Implement SurveyJS and JSON Schema adapters that convert form definitions into L
   - `imagepicker` — single or multiple choice from image options (extract by choice value/text)
   - `imagemap` — clickable regions on an image (extract selected region names)
   - `slider` — numeric slider with min/max/step (extract numeric value)
+  - `signaturepad` — signature image capture (extract as Base64 string)
   - `boolean` — true/false toggle
   - `signature` — skip (handwritten signature, cannot reliably extract as data)
   - `html` — skip (display-only, no data to extract)
@@ -44,6 +45,7 @@ Implement SurveyJS and JSON Schema adapters that convert form definitions into L
 - Include clear instructions about expected value formats per type
 - Clarify canonical output expectations in the prompt and schema notes:
   - Use question `name` as the canonical key even when the form shows a `title`
+  - For `signaturepad`, return a Base64-encoded image string value
   - For `multipletext`, use item `name` keys even if extracted labels use item `title`
   - For matrix types, use canonical row `value` and column `name` (or `value` fallback)
   - For ItemValue arrays (`choices`, `rows`), map display `text` back to canonical `value`
@@ -72,6 +74,7 @@ Implement SurveyJS and JSON Schema adapters that convert form definitions into L
   - `imagepicker` → `z.string()` (single select) or `z.array(z.string())` (multi select)
   - `imagemap` → `z.string()` or `z.array(z.string())` (selected region names)
   - `slider` → `z.number()`
+  - `signaturepad` → `z.string()` (Base64-encoded image string)
   - `signature` → skip
   - `html` → skip
   - `image` → skip

prompts/milestone-5-core-pipeline.md

@@ -32,6 +32,10 @@ Wire up the full end-to-end extraction pipeline in `createExtractor()`.
        - "Return valid JSON only, using canonical schema keys/values (`name`/`value`)"
        - "For each field, include your confidence (0.0-1.0) in a parallel `_confidence` object"
        - "If a field is not visible or unreadable, use null"
+    - If the form contains `signaturepad` elements, append signature-specific guidance:
+       - instruct extraction of the actual handwritten signature marks
+       - require Base64-encoded image string output for signature fields
+       - include signature field names/titles so model can match labels in the scanned document
 
 4. **Call LLM Provider**
    - Convert image to base64 via `imageToBase64()`

src/adapters/__tests__/surveyjs.test.ts

@@ -315,6 +315,22 @@ describe('SurveyJSAdapter.toPrompt', () => {
     expect(adapter.toPrompt({ pages: [] })).toBe('');
     expect(adapter.toPrompt({})).toBe('');
   });
+
+  it('describes signaturepad as base64 string field', () => {
+    const form = {
+      pages: [{
+        name: 'page1',
+        elements: [
+          { type: 'signaturepad', name: 'customerSignature', title: 'Customer Signature', isRequired: true },
+        ],
+      }],
+    };
+
+    const prompt = adapter.toPrompt(form);
+    expect(prompt).toContain('"customerSignature"');
+    expect(prompt).toContain('signature pad');
+    expect(prompt).toContain('base64-encoded image string');
+  });
 });
 
 // ─── toOutputSchema tests ───────────────────────────────────────
@@ -403,6 +419,21 @@ describe('SurveyJSAdapter.toOutputSchema', () => {
     expect(keys).not.toContain('doc');
   });
 
+  it('handles signaturepad as base64 string', () => {
+    const form = {
+      pages: [{
+        name: 'page1',
+        elements: [
+          { type: 'signaturepad', name: 'customerSignature', title: 'Customer Signature', isRequired: true },
+        ],
+      }],
+    };
+
+    const schema = adapter.toOutputSchema(form);
+    expect(schema.safeParse({ customerSignature: 'iVBORw0KGgoAAAANSUhEUgAAAAUA' }).success).toBe(true);
+    expect(schema.safeParse({ customerSignature: 123 }).success).toBe(false);
+  });
+
   it('maps inputType=number to z.number()', () => {
     const form = {
       pages: [{

src/adapters/surveyjs.ts

@@ -313,6 +313,12 @@ function describeElement(el: SurveyElement, index: number): string {
     case 'boolean':
       lines.push('   Type: boolean (true/false)', '   Expected value: true or false');
       break;
+    case 'signaturepad':
+      lines.push(
+        '   Type: signature pad',
+        '   Expected value: a base64-encoded image string of the captured signature',
+      );
+      break;
     case 'matrix': {
       const r = rowLabels(el.rows);
       const c = columnLabels(el.columns);
@@ -435,6 +441,8 @@ function elementToZod(el: SurveyElement): z.ZodTypeAny | null {
       return z.number();
     case 'boolean':
       return z.boolean();
+    case 'signaturepad':
+      return z.string();
     case 'matrix':
       return z.record(z.string());
     case 'matrixdynamic':

src/core/__tests__/extractor.test.ts

@@ -155,6 +155,21 @@ const itemValueTextMappedSurveyDef = {
   ],
 };
 
+const signaturePadSurveyDef = {
+  pages: [
+    {
+      elements: [
+        {
+          type: 'signaturepad',
+          name: 'customerSignature',
+          title: 'Customer Signature',
+          isRequired: true,
+        },
+      ],
+    },
+  ],
+};
+
 const simpleJsonSchemaDef = {
   type: 'object',
   properties: {
@@ -429,6 +444,32 @@ describe('createExtractor', () => {
         },
       });
     });
+
+    it('extracts signaturepad as base64 string value', async () => {
+      const base64Signature = 'iVBORw0KGgoAAAANSUhEUgAAAAUA';
+      const provider = createMockProvider([
+        {
+          content: JSON.stringify({
+            customerSignature: base64Signature,
+          }),
+        },
+      ]);
+
+      const extractor = createExtractor({
+        provider,
+        adapter: 'surveyjs',
+        options: { preprocessImage: false },
+      });
+
+      const result = await extractor.extractFromImage({
+        image: TINY_PNG,
+        formDefinition: signaturePadSurveyDef,
+      });
+
+      expect(result.data).toEqual({
+        customerSignature: base64Signature,
+      });
+    });
   });
 
   describe('retry logic', () => {
@@ -922,6 +963,30 @@ describe('createExtractor', () => {
       expect(call.systemPrompt).toContain('document data extraction assistant');
       expect(call.systemPrompt).toContain('Return valid JSON only');
       expect(call.systemPrompt).toContain('_confidence');
+      expect(call.systemPrompt).not.toContain('signaturepad fields');
+    });
+
+    it('adds signaturepad-specific guidance with field names and labels', async () => {
+      const provider = createMockProvider([
+        { content: JSON.stringify({ customerSignature: 'iVBORw0KGgoAAAANSUhEUgAAAAUA' }) },
+      ]);
+
+      const extractor = createExtractor({
+        provider,
+        adapter: 'surveyjs',
+        options: { preprocessImage: false },
+      });
+
+      await extractor.extractFromImage({
+        image: TINY_PNG,
+        formDefinition: signaturePadSurveyDef,
+      });
+
+      const call = (provider.extractFromImage as ReturnType<typeof vi.fn>).mock.calls[0][0];
+      expect(call.systemPrompt).toContain('For signaturepad fields');
+      expect(call.systemPrompt).toContain('actual handwritten signature marks');
+      expect(call.systemPrompt).toContain('Base64-encoded image string');
+      expect(call.systemPrompt).toContain('"customerSignature" (label: "Customer Signature")');
     });
   });
 });

src/core/extractor.ts

@@ -35,13 +35,77 @@ function resolveAdapter(config: ExtractorConfig): FormAdapter {
   }
 }
 
-const SYSTEM_PROMPT =
+const BASE_SYSTEM_PROMPT =
   'You are a document data extraction assistant. ' +
   'Extract field values from the scanned form image. ' +
   'Return valid JSON only, matching the specified field names exactly. ' +
   'For each field, include your confidence (0.0-1.0) in a parallel "_confidence" object. ' +
   'If a field is not visible or unreadable, use null.';
 
+interface FormElementLike {
+  type?: string;
+  name?: string;
+  title?: string;
+  elements?: FormElementLike[];
+  templateElements?: FormElementLike[];
+}
+
+interface SignatureFieldDescriptor {
+  name: string;
+  label: string;
+}
+
+function collectFormElements(elements: FormElementLike[] | undefined): FormElementLike[] {
+  if (!elements || elements.length === 0) return [];
+
+  const result: FormElementLike[] = [];
+  for (const el of elements) {
+    result.push(el);
+    if (el.elements && el.elements.length > 0) {
+      result.push(...collectFormElements(el.elements));
+    }
+    if (el.templateElements && el.templateElements.length > 0) {
+      result.push(...collectFormElements(el.templateElements));
+    }
+  }
+
+  return result;
+}
+
+function getSignaturePadFields(formDefinition: Record<string, unknown>): SignatureFieldDescriptor[] {
+  const pages = formDefinition.pages as Array<{ elements?: FormElementLike[] }> | undefined;
+  if (!pages || pages.length === 0) return [];
+
+  const fields: SignatureFieldDescriptor[] = [];
+  for (const page of pages) {
+    const elements = collectFormElements(page.elements);
+    for (const el of elements) {
+      if (el.type !== 'signaturepad' || !el.name) continue;
+      fields.push({ name: el.name, label: el.title ?? el.name });
+    }
+  }
+
+  return fields;
+}
+
+function buildSystemPrompt(formDefinition: Record<string, unknown>): string {
+  const signatureFields = getSignaturePadFields(formDefinition);
+  if (signatureFields.length === 0) {
+    return BASE_SYSTEM_PROMPT;
+  }
+
+  const fieldLabels = signatureFields
+    .map((f) => `"${f.name}" (label: "${f.label}")`)
+    .join(', ');
+
+  return (
+    BASE_SYSTEM_PROMPT +
+    ' For signaturepad fields, use field names and labels to identify the signature areas and extract the actual handwritten signature marks.' +
+    ' Return each signature as a Base64-encoded image string without markdown, code fences, or data URL prefixes.' +
+    ` Signature fields: ${fieldLabels}.`
+  );
+}
+
 /**
  * Extracts JSON content from an LLM response that may contain markdown
  * code fences and/or preamble/postamble text.
@@ -160,6 +224,7 @@ export function createExtractor(config: ExtractorConfig) {
       // 3. Generate prompt
       const fieldPrompt = adapter.toPrompt(input.formDefinition);
       const basePrompt = fieldPrompt;
+      const systemPrompt = buildSystemPrompt(input.formDefinition);
 
       // 4. Get output schema for validation
       const outputSchema = adapter.toOutputSchema(input.formDefinition);
@@ -177,7 +242,7 @@ export function createExtractor(config: ExtractorConfig) {
           const response = await provider.extractFromImage({
             image: imageBase64,
             prompt,
-            systemPrompt: SYSTEM_PROMPT,
+            systemPrompt,
           });
 
           const rawContent = response.content;