refactor(memory): redesign extraction/dedup flow and add conflict-aware delete handling (#225)

- refactor session memory pipeline across extractor, deduplicator, and compressor
  - introduce richer dedup decisions and per-memory actions (skip/create/none, merge/delete)
  - improve dedup prompt contracts and memory merge/extraction templates
  - update storage schema/validation/vector backend to support new memory behavior
  - add examples/memory_demo.py and refresh session concept docs for the new flow

Author: zhoujh01

docs/en/concepts/08-session.md

@@ -136,19 +136,20 @@ Messages → LLM Extract → Candidate Memories
               ↓
 Vector Pre-filter → Find Similar Memories
               ↓
-LLM Dedup Decision → CREATE/UPDATE/MERGE/SKIP
+LLM Dedup Decision → candidate(skip/create/none) + item(merge/delete)
               ↓
 Write to AGFS → Vectorize
 ```
 
 ### Dedup Decisions
 
-| Decision | Description |
-|----------|-------------|
-| `CREATE` | New memory, create directly |
-| `UPDATE` | Update existing memory |
-| `MERGE` | Merge multiple memories |
-| `SKIP` | Duplicate, skip |
+| Level | Decision | Description |
+|------|----------|-------------|
+| Candidate | `skip` | Candidate is duplicate, skip and do nothing |
+| Candidate | `create` | Create candidate memory (optionally delete conflicting existing memories first) |
+| Candidate | `none` | Do not create candidate; resolve existing memories by item decisions |
+| Per-existing item | `merge` | Merge candidate content into specified existing memory |
+| Per-existing item | `delete` | Delete specified conflicting existing memory |
 
 ## Storage Structure
 

examples/memory_demo.py

@@ -0,0 +1,179 @@
+"""OpenViking memory dedup demo with REAL LLM decisions (no fake/mock).
+
+This demo focuses on the user preference scenario:
+1) "I like apples."                  -> expected create
+2) "I like strawberries."            -> expected create
+3) "I like Fuji apples."             -> expected merge/none with apple memory
+4) "I do not like fruits anymore."   -> expected delete old positive fruit preferences
+5) repeat negative preference         -> expected skip/none
+
+The script prints commit and find results after each round so you can inspect
+whether memory handling is reasonable.
+
+Usage:
+  export OPENVIKING_CONFIG_FILE=ov.conf
+  python examples/memory_dedup_cases_demo.py
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import shutil
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Iterable, List
+
+from openviking.message.part import TextPart
+from openviking.sync_client import SyncOpenViking
+
+
+@dataclass
+class RoundCase:
+    title: str
+    user_text: str
+    expected: str
+
+
+def _print_section(title: str, body: str = "") -> None:
+    print("\n" + "=" * 80)
+    print(title)
+    if body:
+        print("-" * 80)
+        print(body)
+
+
+def _safe_list(items: Iterable[Any]) -> list:
+    try:
+        return list(items)
+    except Exception:
+        return []
+
+
+def _format_find_result(result: Any, max_items: int = 6) -> str:
+    memories = _safe_list(getattr(result, "memories", []))
+    if not memories:
+        return "(no memory hit)"
+
+    lines: List[str] = []
+    for i, mem in enumerate(memories[:max_items], 1):
+        score = getattr(mem, "score", None)
+        score_s = "n/a" if score is None else f"{float(score):.4f}"
+        abstract = getattr(mem, "abstract", "") or ""
+        uri = getattr(mem, "uri", "")
+        lines.append(f"{i}. score={score_s} | {abstract} | {uri}")
+    return "\n".join(lines)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="OpenViking real-LLM dedup cases demo")
+    parser.add_argument(
+        "--path",
+        default="./ov_data_dedup_cases_demo",
+        help="Fixed demo storage path. This script clears it at startup.",
+    )
+    parser.add_argument(
+        "--wait-timeout",
+        type=float,
+        default=60.0,
+        help="Queue wait timeout in seconds.",
+    )
+    args = parser.parse_args()
+
+    if not os.environ.get("OPENVIKING_CONFIG_FILE"):
+        repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+        cfg = os.path.join(repo_root, "ov.conf")
+        if os.path.exists(cfg):
+            os.environ["OPENVIKING_CONFIG_FILE"] = cfg
+
+    data_path = Path(args.path)
+    if data_path.exists():
+        shutil.rmtree(data_path)
+    data_path.mkdir(parents=True, exist_ok=True)
+
+    client = SyncOpenViking(path=str(data_path))
+    client.initialize()
+
+    try:
+        client.is_healthy()
+        sess_info = client.create_session()
+        session_id = sess_info["session_id"]
+        sess = client.session(session_id)
+
+        rounds = [
+            RoundCase(
+                title="Round 1",
+                user_text=(
+                    "我是一名程序员。"
+                    "我爱吃苹果。"
+                    "我爱吃草莓。"
+                    "我每天早上7点起床。"
+                    "我通勤主要骑共享单车。"
+                    "我习惯在周末整理书桌。"
+                    "我最常用的云盘是Dropbox。"
+                    "我对坚果过敏，尤其是腰果。"
+                    "我最近在学西班牙语。"
+                    "我喜欢在雨天听爵士乐。"
+                    "我的常用笔记软件是Obsidian。"
+                    "我每周三晚上会去游泳。"
+                    "我偏好27英寸的外接显示器。"
+                ),
+                expected="Expected dedup: create multiple unrelated memories",
+            ),
+            RoundCase(
+                title="Round 2",
+                user_text="我爱吃红富士苹果。我是外卖员。",
+                expected="Expected dedup: none+merge (细化苹果偏好)",
+            ),
+            RoundCase(
+                title="Round 3",
+                user_text="我不爱吃水果了，把之前关于喜欢水果的偏好作废。",
+                expected="Expected dedup: delete old positive fruit preferences",
+            ),
+        ]
+
+        queries = [
+            "我喜欢吃什么？",
+            "我是做什么工作的？",
+        ]
+
+        for r in rounds:
+            sess.add_message("user", parts=[TextPart(text=r.user_text)])
+            sess.add_message("assistant", parts=[TextPart(text="收到。")])
+            commit_result = sess.commit()
+
+            _print_section(
+                f"{r.title} commit",
+                body=(f"user={r.user_text}\n{r.expected}\ncommit={commit_result}"),
+            )
+
+            try:
+                client.wait_processed(timeout=args.wait_timeout)
+            except Exception:
+                pass
+
+            for q in queries:
+                try:
+                    result = client.find(q, target_uri="viking://user/memories", limit=8)
+                    _print_section(f"{r.title} find: {q}", body=_format_find_result(result))
+                except Exception as e:
+                    _print_section(f"{r.title} find: {q} (failed)", body=str(e))
+
+        _print_section(
+            "Done",
+            body=(
+                "Check whether the later rounds are dominated by negative fruit preference.\n"
+                "If old positive fruit preferences disappear or stop ranking high, delete likely worked."
+            ),
+        )
+    finally:
+        try:
+            client.close()
+        except Exception:
+            pass
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

openviking/prompts/templates/compression/dedup_decision.yaml

@@ -1,8 +1,8 @@
 metadata:
   id: "compression.dedup_decision"
   name: "Memory Deduplication Decision"
-  description: "Decide whether new memory should CREATE/MERGE/SKIP"
-  version: "2.0.0"
+  description: "Decide candidate action (skip/create/none) and per-memory actions (merge/delete)"
+  version: "3.3.1"
   language: "en"
   category: "compression"
 
@@ -25,25 +25,79 @@ variables:
     required: true
 
 template: |
-  Determine how to handle this candidate memory.
+  You are deciding how to update long-term memory with:
+  1) one candidate memory (new fact)
+  2) existing similar memories (retrieved from store)
 
-  **Candidate Memory**:
-  Abstract: {{ candidate_abstract }}
-  Overview: {{ candidate_overview }}
-  Content: {{ candidate_content }}
+  Candidate memory:
+  - Abstract: {{ candidate_abstract }}
+  - Overview: {{ candidate_overview }}
+  - Content: {{ candidate_content }}
 
-  **Existing Similar Memories**:
+  Existing similar memories:
   {{ existing_memories }}
 
-  Please decide:
-  - SKIP: Candidate memory duplicates existing memories, no need to save
-  - CREATE: This is completely new information, should be created
-  - MERGE: Candidate memory should be merged with existing memories
+  Goal:
+  Keep memory consistent and useful while minimizing destructive edits.
 
-  Return JSON format:
+  Candidate-level decision:
+  - skip:
+    Use only when candidate adds no useful new information (duplicate, paraphrase,
+    or too weak/uncertain). No memory should change.
+  - create:
+    Use when candidate is a valid new memory that should be stored as a separate item.
+    It may optionally delete fully-invalidated existing memories.
+  - none:
+    Use when candidate itself should not be stored, but existing memories should be
+    reconciled with per-item actions.
+
+  Existing-memory per-item action:
+  - merge:
+    Existing memory and candidate are about the same subject and should be unified.
+    Use for refinement, correction, partial conflict, or complementary details.
+  - delete:
+    Existing memory must be removed only if candidate fully invalidates the entire
+    existing memory (not just one sub-part).
+
+  Critical delete boundary:
+  - If conflict is partial (some statements conflict, others remain valid), DO NOT delete.
+    Use merge instead so non-conflicting information is preserved.
+  - Delete only when the whole existing memory is obsolete/invalidated.
+  - Topic/facet mismatch must never be deleted. If candidate is about one facet
+    (for example any single preference facet), existing memories from other facets
+    must be omitted from list (treated as unchanged).
+
+  Decision guidance:
+  - Prefer skip when candidate is redundant.
+  - Prefer none+merge for same-subject updates and partial contradictions.
+  - Prefer create for clearly new independent memory.
+  - If uncertain, choose non-destructive behavior (skip or merge), not delete.
+
+  Practical checklist before emitting each list item:
+  1) Is existing memory about the same topic/facet as candidate?
+  2) If no, do not include it in list.
+  3) If yes and candidate only updates part of it, use merge.
+  4) Use delete only when candidate explicitly invalidates the whole existing memory.
+
+  Hard constraints:
+  - If decision is "skip", do not return "list".
+  - If any list item uses "merge", decision must be "none".
+  - If decision is "create", list can be empty or contain delete items only.
+  - Use uri exactly from existing memories list.
+  - Omit unchanged existing memories from list.
+  - Return JSON only, no prose.
+
+  Return JSON in this exact structure:
   {
-    "decision": "skip|create|merge",
-    "reason": "Decision reason"
+    "decision": "skip|create|none",
+    "reason": "short reason",
+    "list": [
+      {
+        "uri": "<existing memory uri>",
+        "decide": "merge|delete",
+        "reason": "short reason (for delete, explain full invalidation)"
+      }
+    ]
   }
 
 llm_config:

openviking/prompts/templates/compression/memory_extraction.yaml

@@ -2,7 +2,7 @@ metadata:
   id: "compression.memory_extraction"
   name: "Memory Extraction (Three-Level)"
   description: "Extract memories from session context using L0/L1/L2 three-level structure"
-  version: "5.0.0"
+  version: "5.1.0"
   language: "en"
   category: "compression"
 
@@ -90,6 +90,15 @@ template: |
   - Characteristics: Changeable choices, styles
   - Test: Can it be described as "User prefers/likes..."
 
+  ### Preference Granularity (Important)
+  - For category `preferences`, each memory item should represent one independently
+    updatable preference unit (single facet).
+  - Do NOT mix unrelated preference facets in one memory item.
+    Examples of different facets: food, commute, schedule, tools, music, study habits.
+  - If the conversation contains multiple facets, output multiple `preferences` items.
+  - This granularity is required so future updates/conflicts can affect only the
+    relevant memory without damaging unrelated preferences.
+
   **entities** - Entities (continuously existing nouns)
   - Core: Describes "current state of something"
   - Characteristics: Entities with lifecycle (person/project/organization)
@@ -166,6 +175,43 @@ template: |
   ```
   ❌ **Bad**: abstract says "Code preferences" (too general) or "No type hints" (too specific, cannot merge other style preferences)
 
+  ## preferences Granularity Example
+  ❌ **Bad (mixed facets in one memory)**:
+  ```json
+  {
+    "category": "preferences",
+    "abstract": "User preferences: likes apples, commutes by bike, uses Obsidian",
+    "overview": "Mixed food/commute/tool preferences",
+    "content": "User likes apples, usually commutes by bike, and prefers Obsidian."
+  }
+  ```
+
+  ✅ **Good (split by independently updatable facets)**:
+  ```json
+  {
+    "memories": [
+      {
+        "category": "preferences",
+        "abstract": "Food preference: Likes apples",
+        "overview": "## Preference Domain\\n- **Domain**: Food\\n\\n## Specific Preference\\n- Likes apples",
+        "content": "User shows a food preference for apples."
+      },
+      {
+        "category": "preferences",
+        "abstract": "Commute preference: Usually rides a bike",
+        "overview": "## Preference Domain\\n- **Domain**: Commute\\n\\n## Specific Preference\\n- Usually rides a bike",
+        "content": "User usually commutes by bike."
+      },
+      {
+        "category": "preferences",
+        "abstract": "Tool preference: Uses Obsidian for notes",
+        "overview": "## Preference Domain\\n- **Domain**: Tools\\n\\n## Specific Preference\\n- Uses Obsidian for notes",
+        "content": "User prefers Obsidian as note-taking software."
+      }
+    ]
+  }
+  ```
+
   ## entities Example (Merge type)
   ✅ **Good**:
   ```json
@@ -228,8 +274,7 @@ template: |
   - The values of "abstract", "overview", and "content" MUST be written in {{ output_language }} (if output_language is "auto", use the dominant language in recent_messages).
   - Only extract truly valuable personalized information
   - If nothing worth recording, return {"memories": []}
-  - Preferences should be aggregated by topic, similar preferences should be merged into one memory
+  - For preferences, keep each memory as one independently updatable facet; do not combine unrelated facets in one memory
 
 llm_config:
   temperature: 0.0
-