fix(tracker): preserve distinct role identities

Author: CharGrnmn

dedup-tracker.mjs

@@ -6,7 +6,7 @@
  *   - Day-based: data/applications/YYYY-MM-DD.md (preferred)
  *   - Single-file: data/applications.md or applications.md (legacy)
  *
- * Groups by normalized company + fuzzy role match.
+ * Groups by normalized company + conservative role identity.
  * Keeps entry with highest score. If discarded entry had more advanced status,
  * preserves that status. Merges notes.
  *
@@ -21,6 +21,7 @@ import {
   usesDayFiles, ensureDayDir, getHeader, formatAppLine, parseAppLine,
   readAllEntries, writeToDayFiles, listDayFiles, dayFilePath,
 } from './tracker-lib.mjs';
+import { trackerRolesMatch } from './lib/tracker-role-identity.mjs';
 
 const DRY_RUN = process.argv.includes('--dry-run');
 
@@ -62,21 +63,6 @@ function normalizeCompany(name) {
     .trim();
 }
 
-function normalizeRole(role) {
-  return role.toLowerCase()
-    .replace(/[()]/g, ' ')
-    .replace(/\s+/g, ' ')
-    .replace(/[^a-z0-9 /]/g, '')
-    .trim();
-}
-
-function roleMatch(a, b) {
-  const wordsA = normalizeRole(a).split(/\s+/).filter(w => w.length > 3);
-  const wordsB = normalizeRole(b).split(/\s+/).filter(w => w.length > 3);
-  const overlap = wordsA.filter(w => wordsB.some(wb => wb.includes(w) || w.includes(wb)));
-  return overlap.length >= 2;
-}
-
 function parseScore(s) {
   const m = s.replace(/\*\*/g, '').match(/([\d.]+)/);
   return m ? parseFloat(m[1]) : 0;
@@ -115,7 +101,7 @@ for (const [company, companyEntries] of groups) {
 
     for (let j = i + 1; j < companyEntries.length; j++) {
       if (processed.has(j)) continue;
-      if (roleMatch(companyEntries[i].role, companyEntries[j].role)) {
+      if (trackerRolesMatch(companyEntries[i].role, companyEntries[j].role)) {
         cluster.push(companyEntries[j]);
         processed.add(j);
       }

lib/tracker-role-identity.mjs

@@ -0,0 +1,178 @@
+// Tracker deduplication is intentionally stricter than search ranking. A false
+// positive can delete a row or promote another role's status, so uncertain
+// titles remain distinct.
+const TOKEN_ALIASES = new Map([
+  ['backend', ['back', 'end']],
+  ['dev', ['developer']],
+  ['eng', ['engineer']],
+  ['frontend', ['front', 'end']],
+  ['fullstack', ['full', 'stack']],
+  ['jr', ['junior']],
+  ['mgr', ['manager']],
+  ['ml', ['machine', 'learning']],
+  ['programme', ['program']],
+  ['snr', ['senior']],
+  ['sr', ['senior']],
+  ['sre', ['site', 'reliability', 'engineer']],
+  ['swe', ['software', 'engineer']],
+]);
+
+const SENIORITY_TOKENS = new Set([
+  'associate',
+  'distinguished',
+  'junior',
+  'lead',
+  'midlevel',
+  'principal',
+  'senior',
+  'staff',
+]);
+const ROLE_HEAD_TOKENS = new Set([
+  'administrator',
+  'analyst',
+  'architect',
+  'attorney',
+  'consultant',
+  'coordinator',
+  'counsel',
+  'designer',
+  'developer',
+  'director',
+  'editor',
+  'engineer',
+  'executive',
+  'manager',
+  'officer',
+  'producer',
+  'recruiter',
+  'representative',
+  'researcher',
+  'scientist',
+  'specialist',
+  'strategist',
+  'technician',
+  'writer',
+]);
+const ROLE_FAMILY_PREFIX_TOKENS = new Set([
+  'account',
+  'application',
+  'back',
+  'cloud',
+  'customer',
+  'data',
+  'end',
+  'front',
+  'full',
+  'general',
+  'infrastructure',
+  'learning',
+  'machine',
+  'mobile',
+  'platform',
+  'product',
+  'program',
+  'project',
+  'reliability',
+  'sales',
+  'security',
+  'site',
+  'software',
+  'solution',
+  'stack',
+  'success',
+  'system',
+  'technical',
+  'web',
+]);
+
+// This is deliberately an allowlist rather than a general stemmer. Substring
+// or broad suffix matching makes unrelated titles such as product/production
+// and data/database collide.
+const TOKEN_EQUIVALENTS = new Map([
+  ['developers', 'developer'],
+  ['engineers', 'engineer'],
+  ['managers', 'manager'],
+  ['payments', 'payment'],
+  ['platforms', 'platform'],
+  ['products', 'product'],
+  ['programmes', 'program'],
+  ['programs', 'program'],
+  ['projects', 'project'],
+  ['services', 'service'],
+  ['solutions', 'solution'],
+  ['systems', 'system'],
+]);
+
+function normalizedRoleTokens(role) {
+  const rawTokens = String(role || '')
+    .normalize('NFKD')
+    .replace(/[\u0300-\u036f]/g, '')
+    .toLowerCase()
+    .replace(/\bc\+\+(?=$|[^a-z0-9])/g, 'cplusplus')
+    .replace(/\bc#(?=$|[^a-z0-9])/g, 'csharp')
+    .replace(/\bf#(?=$|[^a-z0-9])/g, 'fsharp')
+    .replace(/\.net\b/g, ' dotnet')
+    .replace(/&/g, ' and ')
+    .replace(/['\u2019]/g, '')
+    .replace(/[^a-z0-9]+/g, ' ')
+    .trim()
+    .split(/\s+/)
+    .filter(Boolean);
+  const tokens = [];
+
+  for (let index = 0; index < rawTokens.length; index++) {
+    const token = rawTokens[index];
+    const next = rawTokens[index + 1];
+    if (token === 'entry' && next === 'level') {
+      tokens.push('junior');
+      index++;
+      continue;
+    }
+    if (token === 'mid' && next === 'level') {
+      tokens.push('midlevel');
+      index++;
+      continue;
+    }
+
+    tokens.push(...(TOKEN_ALIASES.get(token) || [token]));
+  }
+
+  for (let index = 0; index < tokens.length; index++) {
+    tokens[index] = TOKEN_EQUIVALENTS.get(tokens[index]) || tokens[index];
+  }
+
+  let seniorityLength = 0;
+  while (seniorityLength < tokens.length && SENIORITY_TOKENS.has(tokens[seniorityLength])) {
+    seniorityLength++;
+  }
+
+  if (seniorityLength > 0) {
+    const tail = tokens.slice(seniorityLength);
+    const headIndex = tail.findIndex((token) => ROLE_HEAD_TOKENS.has(token));
+    const clearlyPositional = headIndex >= 0 && tail
+      .slice(0, headIndex)
+      .every((token) => ROLE_FAMILY_PREFIX_TOKENS.has(token));
+    if (clearlyPositional) tokens.splice(0, seniorityLength);
+  }
+
+  return tokens;
+}
+
+export function trackerRoleIdentity(role) {
+  const tokens = normalizedRoleTokens(role);
+  return {
+    key: tokens.join(':'),
+    tokens,
+  };
+}
+
+export function trackerRolesMatch(leftRole, rightRole) {
+  const left = normalizedRoleTokens(leftRole);
+  const right = normalizedRoleTokens(rightRole);
+
+  if (left.length === 0 || right.length === 0 || left.length !== right.length) {
+    return false;
+  }
+
+  return left.every((token, index) => token === right[index]);
+}

merge-tracker.mjs

@@ -11,7 +11,7 @@
  * - 8-col: num\tdate\tcompany\trole\tstatus\tscore\tpdf\treport (no notes)
  * - Pipe-delimited (markdown table row): | col | col | ... |
  *
- * Dedup: company normalized + role fuzzy match + report number match
+ * Dedup: company normalized + conservative role identity + report number match
  * If duplicate with higher score → update in-place, update report link
  * Validates status against templates/states.yml when present (else built-in labels)
  *
@@ -49,6 +49,7 @@ import {
   writeJobForgeLineage,
 } from './lib/jobforge-lineage.mjs';
 import { compareTrackerAdditionBasenames } from './lib/tracker-addition-order.mjs';
+import { trackerRolesMatch } from './lib/tracker-role-identity.mjs';
 
 const ADDITIONS_DIR = join(PROJECT_DIR, 'batch/tracker-additions');
 const MERGED_DIR = join(ADDITIONS_DIR, 'merged');
@@ -129,35 +130,6 @@ function normalizeCompany(name) {
   return name.toLowerCase().replace(/[^a-z0-9]/g, '');
 }
 
-// Generic seniority + engineering words that appear across most SWE roles
-// and carry no role-specialty signal. A "discriminator" is any remaining
-// word longer than 3 chars (e.g. "Observability", "Telemetry", "Platform").
-const ROLE_STOPWORDS = new Set([
-  'staff', 'senior', 'principal', 'lead', 'junior',
-  'software', 'engineer', 'engineering', 'developer',
-  'backend', 'frontend', 'fullstack', 'full-stack', 'full', 'stack',
-  'technical', 'applied',
-]);
-
-function roleFuzzyMatch(a, b) {
-  // Split on whitespace AND role punctuation (commas, colons, dashes, parens)
-  // so "Staff SWE, Observability K8s" tokenizes past the comma.
-  const split = (s) => s.toLowerCase()
-    .split(/[\s,:\-()\/]+/)
-    .map(w => w.trim())
-    .filter(w => w.length > 3 && !ROLE_STOPWORDS.has(w));
-
-  const wordsA = split(a);
-  const wordsB = split(b);
-
-  // Match on discriminator-word overlap only. Prevents "Staff Software
-  // Engineer, ML Observability" and "Staff Backend Engineer, Adaptive
-  // Telemetry" from colliding (same company, different specialty) while
-  // still collapsing re-evaluations of the same role (same discriminators).
-  const overlap = wordsA.filter(w => wordsB.some(wb => wb.includes(w) || w.includes(wb)));
-  return overlap.length >= 2;
-}
-
 function extractReportNum(reportStr) {
   const m = reportStr.match(/\[(\d+)\]/);
   return m ? parseInt(m[1]) : null;
@@ -329,7 +301,7 @@ for (const file of tsvFiles) {
     const normCompany = normalizeCompany(addition.company);
     duplicate = existingApps.find(app => {
       if (normalizeCompany(app.company) !== normCompany) return false;
-      return roleFuzzyMatch(addition.role, app.role);
+      return trackerRolesMatch(addition.role, app.role);
     });
   }
 

modes/scan.md

@@ -224,15 +224,15 @@ The levels are additive — all are executed, results are merged and deduplicate
    - `scan-history.tsv` → exact URL already seen
    - `pipeline.md` → exact URL already in pending or processed
 
-   **Layer 2 — Company + role fuzzy match (catches reposts with new URLs):**
-   - all day files in `data/applications/` → normalize company name (lowercase, strip non-alphanumeric) + fuzzy role match (2+ significant words in common, words > 3 chars). This is the same logic used in `dedup-tracker.mjs` and `merge-tracker.mjs`.
-   - `scan-history.tsv` → same fuzzy match against company + title columns (not just URL). A role reposted on a new URL but with the same company and similar title is a duplicate.
-   - `pipeline.md` → same fuzzy match against company + title in pending items that include metadata (format: `- [ ] {url} | {company} | {title}`)
+   **Layer 2 — Company + conservative role identity (catches true reposts with new URLs):**
+   - all day files in `data/applications/` → normalize company name (lowercase, strip non-alphanumeric) + a conservative scan-layer role comparison. This prompt-driven prefilter does not execute the tracker matcher. Keep the candidate whenever identity is uncertain; `dedup-tracker.mjs` and `merge-tracker.mjs` apply their shared deterministic tracker identity during settlement.
+   - `scan-history.tsv` → apply the same conservative identity policy to company + title columns (not just URL). A role reposted on a new URL is a duplicate only when the normalized role identities agree.
+   - `pipeline.md` → apply the same conservative identity policy to company + title in pending items that include metadata (format: `- [ ] {url} | {company} | {title}`)
 
-   **Fuzzy match rules:**
+   **Role identity rules:**
    - Normalize company: `company.toLowerCase().replace(/[^a-z0-9]/g, '')`
-   - Fuzzy role match: split both titles into words > 3 chars, match if 2+ words overlap (substring match, case-insensitive). E.g., "Senior AI Engineer" and "Staff AI Engineer" share "engineer" — only 1 overlap, not a match. But "AI Platform Engineer" and "AI Platform Eng" share "platform" + partial "engineer" — match.
-   - When a fuzzy match is found but the URL is new, log it as `skipped_repost` (not `skipped_dup`) with a note referencing the original entry number.
+   - Preserve every normalized role word and its order; never apply global role stopwords or derivational stemming such as `engineering`→`engineer` or `management`→`manager`. Preserve semantic punctuation before tokenization: `C++`, `C#`, `F#`, and `.NET` remain distinct from `C`, `F`, and `NET`. Location or work-mode words such as `US`, `United States`, `remote`, `hybrid`, and `New York` remain identity-bearing. Do not use substring or shared generic-title overlap. Remove only clearly positional seniority prefixes; never globally erase ambiguous words such as `lead`, `staff`, or `associate`. Short specialty tokens remain significant: `UI Engineer` and `UX Engineer` are distinct; `Product Manager, Cash Platform` and `AI Product Manager, Professional Services` are distinct; `Senior Product Managers, Payments` and `Product Manager - Payment` match after safe plural and seniority normalization.
+   - When a role-identity match is found but the URL is new, log it as `skipped_repost` (not `skipped_dup`) with a note referencing the original entry number.
 
 10. **Balance the eligible, deduplicated shortlist** using `diversity_policy`:
    - Write the normalized candidate array to `/tmp/jobforge-scan-candidates-{YYYY-MM-DD}.json`, including posting-derived `country_code`, `relevance_score`, `region`, and `location_status`.

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "job-forge",
-  "version": "2.14.69",
+  "version": "2.14.70",
   "description": "AI-powered job search pipeline for Codex and OpenCode",
   "type": "module",
   "bin": {