diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cc1cdbe7..c93f0ebb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -53,6 +53,7 @@ jobs: echo "CS_CLIENT_ACCESS_KEY=${{ secrets.CS_CLIENT_ACCESS_KEY }}" >> ./packages/stack/.env echo "SUPABASE_URL=${{ secrets.SUPABASE_URL }}" >> ./packages/stack/.env echo "SUPABASE_ANON_KEY=${{ secrets.SUPABASE_ANON_KEY }}" >> ./packages/stack/.env + echo "DATABASE_URL=${{ secrets.DATABASE_URL }}" >> ./packages/stack/.env echo "CS_ZEROKMS_HOST=https://ap-southeast-2.aws.zerokms.cipherstashmanaged.net" >> ./packages/stack/.env echo "CS_CTS_HOST=https://ap-southeast-2.aws.cts.cipherstashmanaged.net" >> ./packages/stack/.env diff --git a/docs/reference/searchable-encryption-postgres.md b/docs/reference/searchable-encryption-postgres.md index 407bd950..cfcdc8a8 100644 --- a/docs/reference/searchable-encryption-postgres.md +++ b/docs/reference/searchable-encryption-postgres.md @@ -7,6 +7,8 @@ This reference guide outlines the different query patterns you can use to search - [Prerequisites](#prerequisites) - [What is EQL?](#what-is-eql) - [Setting up your schema](#setting-up-your-schema) +- [The `encryptQuery` function](#the-encryptquery-function) + - [Formatting encrypted query results with `returnType`](#formatting-encrypted-query-results-with-returntype) - [Search capabilities](#search-capabilities) - [JSONB queries with searchableJson (recommended)](#jsonb-queries-with-searchablejson-recommended) - [JSONPath selector queries](#jsonpath-selector-queries) @@ -79,8 +81,10 @@ The `encryptQuery` function is used to create encrypted query terms for use in S | `value` | The value to search for | | `column` | The column to search in | | `table` | The table to search in | +| `queryType` | _(optional)_ The query type — auto-inferred from the column's indexes when omitted | +| `returnType` | _(optional)_ The output format — `'eql'` (default), `'composite-literal'`, or `'escaped-composite-literal'` | -**Batch query** — pass an array of objects, each with the properties above. +**Batch query** — pass an array of objects, each with the properties above (including `value`). Example (single query): @@ -112,6 +116,58 @@ if (terms.failure) { console.log(terms.data) // array of encrypted query terms ``` +### Formatting encrypted query results with `returnType` + +By default, `encryptQuery` returns an `Encrypted` object (the raw EQL JSON payload). You can change the output format using the `returnType` option: + +| `returnType` | Return type | Description | +|---|---|---| +| `'eql'` (default) | `Encrypted` object | Raw EQL JSON payload. Use with parameterized queries (`$1`) or ORMs that accept JSON objects. | +| `'composite-literal'` | `string` | PostgreSQL composite literal format `("json")`. Use with Supabase `.eq()` or other APIs that require a string value. | +| `'escaped-composite-literal'` | `string` | Escaped composite literal `"(\"json\")"`. Use when the query string will be embedded inside another string or JSON value. | + +The return type of `encryptQuery` is `EncryptedQueryResult`, which is `Encrypted | string | null` depending on the `returnType` and whether the input was `null`. + +**Single query with `returnType`:** + +```typescript +const term = await client.encryptQuery('user@example.com', { + column: schema.email, + table: schema, + queryType: 'equality', + returnType: 'composite-literal', +}) + +if (term.failure) { + // Handle the error +} + +// term.data is a string in composite literal format +await supabase.from('users').select().eq('email_encrypted', term.data) +``` + +**Batch query with `returnType`:** + +Each term in a batch can have its own `returnType`: + +```typescript +const terms = await client.encryptQuery([ + { + value: 'user@example.com', + column: schema.email, + table: schema, + queryType: 'equality', + returnType: 'composite-literal', // returns a string + }, + { + value: 'alice', + column: schema.email, + table: schema, + queryType: 'freeTextSearch', // returns an Encrypted object (default) + }, +]) +``` + ## Search capabilities ### JSONB queries with searchableJson (recommended) @@ -257,27 +313,63 @@ console.log(terms.data) // array of encrypted query terms #### Using JSONB queries in SQL -To use encrypted JSONB query terms in PostgreSQL queries, specify `returnType: 'composite-literal'` to get the terms formatted for direct use in SQL: +To use encrypted JSONB query terms in PostgreSQL queries, you can either use the default `Encrypted` object with parameterized queries, or use `returnType: 'composite-literal'` to get a string formatted for direct use with Supabase or similar APIs. + +**With parameterized queries (default `returnType`):** ```typescript -const term = await client.encryptQuery([{ - value: '$.user.email', +const term = await client.encryptQuery('$.user.email', { column: documents.metadata, table: documents, - returnType: 'composite-literal', -}]) +}) if (term.failure) { // Handle the error } -// Use the encrypted term in a PostgreSQL query -const result = await client.query( +// Pass the EQL object as a parameterized query value +const result = await pgClient.query( 'SELECT * FROM documents WHERE cs_ste_vec_v2(metadata_encrypted) @> $1', - [term.data[0]] + [term.data] ) ``` +**With Supabase or string-based APIs (`returnType: 'composite-literal'`):** + +```typescript +const term = await client.encryptQuery('$.user.email', { + column: documents.metadata, + table: documents, + returnType: 'composite-literal', +}) + +if (term.failure) { + // Handle the error +} + +// term.data is a string — use directly with .eq(), .contains(), etc. +await supabase.from('documents').select().contains('metadata_encrypted', term.data) +``` + +This also works with batch queries — each term can specify its own `returnType`: + +```typescript +const terms = await client.encryptQuery([ + { + value: '$.user.email', + column: documents.metadata, + table: documents, + returnType: 'composite-literal', + }, + { + value: { role: 'admin' }, + column: documents.metadata, + table: documents, + returnType: 'composite-literal', + }, +]) +``` + #### Advanced: Explicit query types For advanced use cases, you can specify the query type explicitly instead of relying on auto-inference: diff --git a/packages/stack/README.md b/packages/stack/README.md index 83b9a66e..5387e7ee 100644 --- a/packages/stack/README.md +++ b/packages/stack/README.md @@ -264,37 +264,92 @@ const terms = [ const results = await client.encryptQuery(terms) ``` -### PostgreSQL / Drizzle Integration Pattern +### Query Result Formatting (`returnType`) + +By default `encryptQuery` returns an `Encrypted` object (the raw EQL JSON payload). Use `returnType` to change the output format: -Encrypted data is stored as an [EQL](https://github.com/cipherstash/encrypt-query-language) JSON payload. Install the EQL extension in PostgreSQL to enable searchable queries, then store encrypted data in `eql_v2_encrypted` columns: +| `returnType` | Output | Use case | +|---|---|---| +| `'eql'` (default) | `Encrypted` object | Parameterized queries, ORMs accepting JSON | +| `'composite-literal'` | `string` | Supabase `.eq()`, string-based APIs | +| `'escaped-composite-literal'` | `string` | Embedding inside another string or JSON value | -```sql -CREATE TABLE users ( - id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, - email eql_v2_encrypted -); +```typescript +// Get a composite literal string for use with Supabase +const term = await client.encryptQuery("alice@example.com", { + column: users.email, + table: users, + queryType: "equality", + returnType: "composite-literal", +}) + +// term.data is a string — use directly with .eq() +await supabase.from("users").select().eq("email", term.data) ``` +Each term in a batch can have its own `returnType`. + +### PostgreSQL / Drizzle Integration Pattern + +Encrypted data is stored as an [EQL](https://github.com/cipherstash/encrypt-query-language) JSON payload. Install the EQL extension in PostgreSQL to enable searchable queries, then store encrypted data in `eql_v2_encrypted` columns. + +The `@cipherstash/stack/drizzle` module provides `encryptedType` for defining encrypted columns and `createEncryptionOperators` for querying them: + ```typescript -import { eq } from "drizzle-orm" -import { pgTable, serial, jsonb } from "drizzle-orm/pg-core" +import { pgTable, integer, timestamp } from "drizzle-orm/pg-core" +import { encryptedType, extractEncryptionSchema, createEncryptionOperators } from "@cipherstash/stack/drizzle" +import { Encryption } from "@cipherstash/stack" +// Define schema with encrypted columns const usersTable = pgTable("users", { - id: serial("id").primaryKey(), - email: jsonb("email").notNull(), + id: integer("id").primaryKey().generatedAlwaysAsIdentity(), + email: encryptedType("email", { + equality: true, + freeTextSearch: true, + orderAndRange: true, + }), + profile: encryptedType<{ name: string; bio: string }>("profile", { + dataType: "json", + searchableJson: true, + }), }) -// Insert encrypted data -await db.insert(usersTable).values({ email: encrypted.data }) +// Initialize +const usersSchema = extractEncryptionSchema(usersTable) +const client = await Encryption({ schemas: [usersSchema] }) +const ops = createEncryptionOperators(client) -// Search with encrypted query -const encQuery = await client.encryptQuery("alice@example.com", { - column: users.email, - table: users, - queryType: "equality", -}) +// Query with auto-encrypting operators +const results = await db.select().from(usersTable) + .where(await ops.eq(usersTable.email, "alice@example.com")) + +// JSONB queries on encrypted JSON columns +const jsonResults = await db.select().from(usersTable) + .where(await ops.jsonbPathExists(usersTable.profile, "$.bio")) ``` +#### Drizzle `encryptedType` Config Options + +| Option | Type | Description | +|---|---|---| +| `dataType` | `"string"` \| `"number"` \| `"json"` | Plaintext data type (default: `"string"`) | +| `equality` | `boolean` \| `TokenFilter[]` | Enable equality index | +| `freeTextSearch` | `boolean` \| `MatchIndexOpts` | Enable free-text search index | +| `orderAndRange` | `boolean` | Enable ORE index for sorting/range queries | +| `searchableJson` | `boolean` | Enable JSONB path queries (requires `dataType: "json"`) | + +#### Drizzle JSONB Operators + +For columns with `searchableJson: true`, three JSONB operators are available: + +| Operator | Description | +|---|---| +| `jsonbPathExists(col, selector)` | Check if a JSONB path exists (boolean, use in `WHERE`) | +| `jsonbPathQueryFirst(col, selector)` | Extract first value at a JSONB path | +| `jsonbGet(col, selector)` | Get value using the JSONB `->` operator | + +These operators encrypt the JSON path selector using the `steVecSelector` query type and cast it to `eql_v2_encrypted` for use with the EQL PostgreSQL functions. + ## Identity-Aware Encryption Lock encryption to a specific user by requiring a valid JWT for decryption. @@ -535,7 +590,7 @@ function Encryption(config: EncryptionClientConfig): Promise |----|------|-----| | `encrypt` | `(plaintext, { column, table })` | `EncryptOperation` (thenable) | | `decrypt` | `(encryptedData)` | `DecryptOperation` (thenable) | -| `encryptQuery` | `(plaintext, { column, table, queryType? })` | `EncryptQueryOperation` (thenable) | +| `encryptQuery` | `(plaintext, { column, table, queryType?, returnType? })` | `EncryptQueryOperation` (thenable) | | `encryptQuery` | `(terms: ScalarQueryTerm[])` | `BatchEncryptQueryOperation` (thenable) | | `encryptModel` | `(model, table)` | `EncryptModelOperation` (thenable) | | `decryptModel` | `(encryptedModel)` | `DecryptModelOperation` (thenable) | diff --git a/packages/stack/__tests__/drizzle-operators-jsonb.test.ts b/packages/stack/__tests__/drizzle-operators-jsonb.test.ts new file mode 100644 index 00000000..822efce2 --- /dev/null +++ b/packages/stack/__tests__/drizzle-operators-jsonb.test.ts @@ -0,0 +1,212 @@ +import type { EncryptionClient } from '@/encryption/ffi' +import { pgTable } from 'drizzle-orm/pg-core' +import { PgDialect } from 'drizzle-orm/pg-core' +import { describe, expect, it, vi } from 'vitest' +import { + createEncryptionOperators, + EncryptionOperatorError, + encryptedType, +} from '@/drizzle' + +const ENCRYPTED_VALUE = '{"v":"encrypted-value"}' + +function createMockEncryptionClient() { + const encryptQuery = vi.fn(async (termsOrValue: unknown) => { + if (Array.isArray(termsOrValue)) { + return { data: termsOrValue.map(() => ENCRYPTED_VALUE) } + } + return { data: ENCRYPTED_VALUE } + }) + + return { + client: { encryptQuery } as unknown as EncryptionClient, + encryptQuery, + } +} + +function setup() { + const { client, encryptQuery } = createMockEncryptionClient() + const encryptionOps = createEncryptionOperators(client) + const dialect = new PgDialect() + return { client, encryptQuery, encryptionOps, dialect } +} + +const docsTable = pgTable('json_docs', { + metadata: encryptedType>('metadata', { + dataType: 'json', + searchableJson: true, + }), + noJsonConfig: encryptedType('no_json_config', { + equality: true, + }), +}) + +describe('createEncryptionOperators JSONB selector typing', () => { + it('casts jsonbPathQueryFirst selector params to eql_v2_encrypted', async () => { + const { encryptQuery, encryptionOps, dialect } = setup() + + const condition = await encryptionOps.jsonbPathQueryFirst( + docsTable.metadata, + '$.profile.email', + ) + const query = dialect.sqlToQuery(condition) + + expect(query.sql).toMatch( + /eql_v2\.jsonb_path_query_first\([^,]+,\s*\$\d+::eql_v2_encrypted\)/, + ) + expect(query.params).toHaveLength(1) + expect(query.params[0]).toContain('encrypted-value') + expect(encryptQuery).toHaveBeenCalledTimes(1) + expect(encryptQuery.mock.calls[0]?.[0]).toMatchObject([ + { queryType: 'steVecSelector' }, + ]) + }) + + it('casts jsonbPathExists selector params to eql_v2_encrypted', async () => { + const { encryptQuery, encryptionOps, dialect } = setup() + + const condition = await encryptionOps.jsonbPathExists( + docsTable.metadata, + '$.profile.email', + ) + const query = dialect.sqlToQuery(condition) + + expect(query.sql).toMatch( + /eql_v2\.jsonb_path_exists\([^,]+,\s*\$\d+::eql_v2_encrypted\)/, + ) + expect(query.params).toHaveLength(1) + expect(query.params[0]).toContain('encrypted-value') + expect(encryptQuery).toHaveBeenCalledTimes(1) + expect(encryptQuery.mock.calls[0]?.[0]).toMatchObject([ + { queryType: 'steVecSelector' }, + ]) + }) + + it('casts jsonbGet selector params to eql_v2_encrypted', async () => { + const { encryptQuery, encryptionOps, dialect } = setup() + + const condition = await encryptionOps.jsonbGet( + docsTable.metadata, + '$.profile.email', + ) + const query = dialect.sqlToQuery(condition) + + expect(query.sql).toMatch(/->\s*\$\d+::eql_v2_encrypted/) + expect(query.params).toHaveLength(1) + expect(query.params[0]).toContain('encrypted-value') + expect(encryptQuery).toHaveBeenCalledTimes(1) + expect(encryptQuery.mock.calls[0]?.[0]).toMatchObject([ + { queryType: 'steVecSelector' }, + ]) + }) +}) + +describe('JSONB operator error paths', () => { + it('throws EncryptionOperatorError when column lacks searchableJson config', () => { + const { encryptionOps } = setup() + + expect(() => + encryptionOps.jsonbPathQueryFirst(docsTable.noJsonConfig, '$.path'), + ).toThrow(EncryptionOperatorError) + + expect(() => + encryptionOps.jsonbPathQueryFirst(docsTable.noJsonConfig, '$.path'), + ).toThrow(/searchableJson/) + }) + + it('throws EncryptionOperatorError for jsonbPathExists without searchableJson', () => { + const { encryptionOps } = setup() + + expect(() => + encryptionOps.jsonbPathExists(docsTable.noJsonConfig, '$.path'), + ).toThrow(EncryptionOperatorError) + }) + + it('throws EncryptionOperatorError for jsonbGet without searchableJson', () => { + const { encryptionOps } = setup() + + expect(() => + encryptionOps.jsonbGet(docsTable.noJsonConfig, '$.path'), + ).toThrow(EncryptionOperatorError) + }) + + it('error includes column name and operator context', () => { + const { encryptionOps } = setup() + + try { + encryptionOps.jsonbPathQueryFirst(docsTable.noJsonConfig, '$.path') + expect.fail('Should have thrown') + } catch (error) { + expect(error).toBeInstanceOf(EncryptionOperatorError) + const opError = error as EncryptionOperatorError + expect(opError.context?.columnName).toBe('no_json_config') + expect(opError.context?.operator).toBe('jsonbPathQueryFirst') + } + }) +}) + +describe('JSONB batched operations', () => { + it('batches jsonbPathQueryFirst and jsonbGet in encryptionOps.and()', async () => { + const { encryptQuery, encryptionOps, dialect } = setup() + + const condition = await encryptionOps.and( + encryptionOps.jsonbPathQueryFirst(docsTable.metadata, '$.profile.email'), + encryptionOps.jsonbGet(docsTable.metadata, '$.profile.name'), + ) + const query = dialect.sqlToQuery(condition) + + expect(query.sql).toContain('eql_v2.jsonb_path_query_first') + expect(query.sql).toContain('->') + // Verify batch encryption happened (at least one call with 2 terms) + expect( + encryptQuery.mock.calls.some( + (call: unknown[]) => Array.isArray(call[0]) && call[0].length === 2, + ), + ).toBe(true) + }) + + it('batches jsonbPathExists and jsonbPathQueryFirst in encryptionOps.or()', async () => { + const { encryptQuery, encryptionOps, dialect } = setup() + + const condition = await encryptionOps.or( + encryptionOps.jsonbPathExists(docsTable.metadata, '$.profile.email'), + encryptionOps.jsonbPathQueryFirst(docsTable.metadata, '$.profile.name'), + ) + const query = dialect.sqlToQuery(condition) + + expect(query.sql).toContain('eql_v2.jsonb_path_exists') + expect(query.sql).toContain('eql_v2.jsonb_path_query_first') + // Verify batch encryption happened (at least one call with 2 terms) + expect( + encryptQuery.mock.calls.some( + (call: unknown[]) => Array.isArray(call[0]) && call[0].length === 2, + ), + ).toBe(true) + }) + + it('generates SQL combining conditions with AND', async () => { + const { encryptionOps, dialect } = setup() + + const condition = await encryptionOps.and( + encryptionOps.jsonbPathQueryFirst(docsTable.metadata, '$.a'), + encryptionOps.jsonbPathExists(docsTable.metadata, '$.b'), + ) + const query = dialect.sqlToQuery(condition) + + // AND combines conditions + expect(query.sql).toContain(' and ') + }) + + it('generates SQL combining conditions with OR', async () => { + const { encryptionOps, dialect } = setup() + + const condition = await encryptionOps.or( + encryptionOps.jsonbPathQueryFirst(docsTable.metadata, '$.a'), + encryptionOps.jsonbPathExists(docsTable.metadata, '$.b'), + ) + const query = dialect.sqlToQuery(condition) + + // OR combines conditions + expect(query.sql).toContain(' or ') + }) +}) diff --git a/packages/stack/__tests__/encrypt-query-searchable-json.test.ts b/packages/stack/__tests__/encrypt-query-searchable-json.test.ts index c3611509..13b46176 100644 --- a/packages/stack/__tests__/encrypt-query-searchable-json.test.ts +++ b/packages/stack/__tests__/encrypt-query-searchable-json.test.ts @@ -403,6 +403,85 @@ describe('searchableJson with returnType formatting', () => { }, 30000) }) +describe('single-value returnType', () => { + let protectClient: EncryptionClient + + beforeAll(async () => { + protectClient = await Encryption({ schemas: [jsonbSchema] }) + }) + + it('returns composite-literal for selector', async () => { + const result = await protectClient.encryptQuery('$.user.email', { + column: jsonbSchema.metadata, + table: jsonbSchema, + queryType: 'searchableJson', + returnType: 'composite-literal', + }) + + const data = unwrapResult(result) + expect(typeof data).toBe('string') + expect(data).toMatch(/^\(".*"\)$/) + }, 30000) + + it('returns composite-literal for term', async () => { + const result = await protectClient.encryptQuery({ role: 'admin' }, { + column: jsonbSchema.metadata, + table: jsonbSchema, + queryType: 'searchableJson', + returnType: 'composite-literal', + }) + + const data = unwrapResult(result) + expect(typeof data).toBe('string') + expect(data).toMatch(/^\(".*"\)$/) + }, 30000) + + it('returns escaped-composite-literal for selector', async () => { + const result = await protectClient.encryptQuery('$.user.email', { + column: jsonbSchema.metadata, + table: jsonbSchema, + queryType: 'searchableJson', + returnType: 'escaped-composite-literal', + }) + + const data = unwrapResult(result) + expect(typeof data).toBe('string') + expect(data as string).toMatch(/^"\(.*\)"$/) + // JSON.parse should yield the composite-literal format + const parsed = JSON.parse(data as string) + expect(parsed).toMatch(/^\(.*\)$/) + }, 30000) + + it('returns escaped-composite-literal for term', async () => { + const result = await protectClient.encryptQuery({ role: 'admin' }, { + column: jsonbSchema.metadata, + table: jsonbSchema, + queryType: 'searchableJson', + returnType: 'escaped-composite-literal', + }) + + const data = unwrapResult(result) + expect(typeof data).toBe('string') + expect(data as string).toMatch(/^"\(.*\)"$/) + const parsed = JSON.parse(data as string) + expect(parsed).toMatch(/^\(.*\)$/) + }, 30000) + + it('returns Encrypted object when returnType is omitted', async () => { + const result = await protectClient.encryptQuery('$.user.email', { + column: jsonbSchema.metadata, + table: jsonbSchema, + queryType: 'searchableJson', + }) + + const data = unwrapResult(result) as any + expect(typeof data).toBe('object') + expect(data).toHaveProperty('i') + expect(data.i).toHaveProperty('t') + expect(data.i).toHaveProperty('c') + }, 30000) +}) + describe('searchableJson with LockContext', () => { let protectClient: EncryptionClient diff --git a/packages/stack/__tests__/encrypt-query.test.ts b/packages/stack/__tests__/encrypt-query.test.ts index 4533cb1f..208364f3 100644 --- a/packages/stack/__tests__/encrypt-query.test.ts +++ b/packages/stack/__tests__/encrypt-query.test.ts @@ -725,6 +725,84 @@ describe('encryptQuery', () => { }, 30000) }) + describe('single-value returnType formatting', () => { + it('returns Encrypted by default (no returnType)', async () => { + const result = await protectClient.encryptQuery('test@example.com', { + column: users.email, + table: users, + queryType: 'equality', + }) + + const data = unwrapResult(result) + + expect(data).toMatchObject({ + i: { t: 'users', c: 'email' }, + v: 2, + }) + expect(typeof data).toBe('object') + }, 30000) + + it('returns composite-literal format when specified', async () => { + const result = await protectClient.encryptQuery('test@example.com', { + column: users.email, + table: users, + queryType: 'equality', + returnType: 'composite-literal', + }) + + const data = unwrapResult(result) + + expect(typeof data).toBe('string') + // Format: ("json") + expect(data).toMatch(/^\(".*"\)$/) + }, 30000) + + it('returns escaped-composite-literal format when specified', async () => { + const result = await protectClient.encryptQuery('test@example.com', { + column: users.email, + table: users, + queryType: 'equality', + returnType: 'escaped-composite-literal', + }) + + const data = unwrapResult(result) + + expect(typeof data).toBe('string') + // Format: "(\"json\")" - outer quotes with escaped inner quotes + expect(data).toMatch(/^"\(.*\)"$/) + }, 30000) + + it('returns eql format when explicitly specified', async () => { + const result = await protectClient.encryptQuery('test@example.com', { + column: users.email, + table: users, + queryType: 'equality', + returnType: 'eql', + }) + + const data = unwrapResult(result) + + expect(data).toMatchObject({ + i: { t: 'users', c: 'email' }, + v: 2, + }) + expect(typeof data).toBe('object') + }, 30000) + + it('handles null value with composite-literal returnType', async () => { + const result = await protectClient.encryptQuery(null, { + column: users.email, + table: users, + queryType: 'equality', + returnType: 'composite-literal', + }) + + const data = unwrapResult(result) + + expect(data).toBeNull() + }, 30000) + }) + describe('LockContext support', () => { it('single query with LockContext calls getLockContext', async () => { const mockLockContext = createMockLockContext() diff --git a/packages/stack/__tests__/searchable-json-pg.test.ts b/packages/stack/__tests__/searchable-json-pg.test.ts new file mode 100644 index 00000000..9c884061 --- /dev/null +++ b/packages/stack/__tests__/searchable-json-pg.test.ts @@ -0,0 +1,1293 @@ +import 'dotenv/config' +import { encryptedColumn, encryptedTable } from '@/schema' +import postgres from 'postgres' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { Encryption } from '../src' +import { LockContext } from '../src/identity' + +if (!process.env.DATABASE_URL) { + throw new Error('Missing env.DATABASE_URL') +} + +// Disable prepared statements — required for pooled connections (PgBouncer in transaction mode) +const sql = postgres(process.env.DATABASE_URL, { prepare: false }) + +const table = encryptedTable('protect-ci-jsonb', { + metadata: encryptedColumn('metadata').searchableJson(), +}) + +const TEST_RUN_ID = `test-run-${Date.now()}-${Math.random().toString(36).slice(2, 8)}` +const userJwt = process.env.USER_JWT + +type EncryptionClient = Awaited> +let protectClient: EncryptionClient + +// ─── Helpers ───────────────────────────────────────────────────────── + +async function insertRow(plaintext: any) { + const encrypted = await protectClient.encryptModel( + { metadata: plaintext }, + table, + ) + if (encrypted.failure) throw new Error(encrypted.failure.message) + + const [inserted] = await sql` + INSERT INTO "protect-ci-jsonb" (metadata, test_run_id) + VALUES (${sql.json(encrypted.data.metadata)}::eql_v2_encrypted, ${TEST_RUN_ID}) + RETURNING id + ` + return { id: inserted.id, encrypted } +} + +async function verifyRow(row: any, expected: any) { + expect(row).toBeDefined() + const decrypted = await protectClient.decryptModel({ metadata: row.metadata }) + if (decrypted.failure) throw new Error(decrypted.failure.message) + expect(decrypted.data.metadata).toEqual(expected) +} + +async function encryptQueryTerm( + value: any, + queryType: 'steVecSelector' | 'steVecTerm' | 'searchableJson', + returnType: + | 'composite-literal' + | 'escaped-composite-literal' = 'composite-literal', +) { + const result = await protectClient.encryptQuery(value, { + column: table.metadata, + table: table, + queryType, + returnType, + }) + if (result.failure) throw new Error(result.failure.message) + return result.data +} + +beforeAll(async () => { + protectClient = await Encryption({ schemas: [table] }) + + await sql` + CREATE TABLE IF NOT EXISTS "protect-ci-jsonb" ( + id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, + metadata eql_v2_encrypted, + test_run_id TEXT + ) + ` +}, 30000) + +afterAll(async () => { + await sql`DELETE FROM "protect-ci-jsonb" WHERE test_run_id = ${TEST_RUN_ID}` + await sql.end() +}, 30000) + +describe('searchableJson postgres integration', () => { + // ─── Storage: encrypt → insert → select → decrypt ────────────────── + + describe('storage: encrypt → insert → select → decrypt', () => { + it('round-trips a flat JSON object', async () => { + const plaintext = { user: { email: 'flat-rt@test.com' }, role: 'admin' } + const { id } = await insertRow(plaintext) + + const rows = await sql` + SELECT id, (metadata).data as metadata FROM "protect-ci-jsonb" + WHERE id = ${id} + ` + + expect(rows).toHaveLength(1) + await verifyRow(rows[0], plaintext) + }, 30000) + + it('round-trips nested JSON with arrays', async () => { + const plaintext = { + user: { + profile: { role: 'admin', permissions: ['read', 'write'] }, + tags: [{ name: 'vip' }, { name: 'beta' }], + }, + items: [{ id: 1, name: 'widget' }], + } + const { id } = await insertRow(plaintext) + + const rows = await sql` + SELECT id, (metadata).data as metadata FROM "protect-ci-jsonb" + WHERE id = ${id} + ` + + expect(rows).toHaveLength(1) + await verifyRow(rows[0], plaintext) + }, 30000) + + it('round-trips null values', async () => { + const encrypted = await protectClient.encrypt(null, { + column: table.metadata, + table: table, + }) + + if (encrypted.failure) throw new Error(encrypted.failure.message) + expect(encrypted.data).toBeNull() + + const [inserted] = await sql` + INSERT INTO "protect-ci-jsonb" (metadata, test_run_id) + VALUES (NULL, ${TEST_RUN_ID}) + RETURNING id + ` + + const rows = await sql` + SELECT id, (metadata).data as metadata FROM "protect-ci-jsonb" + WHERE id = ${inserted.id} + ` + + expect(rows).toHaveLength(1) + expect(rows[0].metadata).toBeNull() + }, 30000) + }) + + // ─── jsonb_path_query: path-based selector queries ───────────────── + + describe('jsonb_path_query: path-based selector queries', () => { + it('finds row by simple top-level path ($.role)', async () => { + const plaintext = { role: 'path-toplevel-test', extra: 'data' } + const { id } = await insertRow(plaintext) + + const selectorTerm = await encryptQueryTerm('$.role', 'steVecSelector') + + const rows = await sql` + SELECT id, (metadata).data as metadata + FROM "protect-ci-jsonb" t, + eql_v2.jsonb_path_query(t.metadata, ${selectorTerm}::eql_v2_encrypted) as result + WHERE t.test_run_id = ${TEST_RUN_ID} + ` + + expect(rows.length).toBeGreaterThanOrEqual(1) + const matchingRow = rows.find((r) => r.id === id) + expect(matchingRow).toBeDefined() + await verifyRow(matchingRow!, plaintext) + }, 30000) + + it('finds row by nested path ($.user.email)', async () => { + const plaintext = { + user: { email: 'nested-path@test.com' }, + type: 'nested-path', + } + const { id } = await insertRow(plaintext) + + const selectorTerm = await encryptQueryTerm( + '$.user.email', + 'steVecSelector', + ) + + const rows = await sql` + SELECT id, (metadata).data as metadata + FROM "protect-ci-jsonb" t, + eql_v2.jsonb_path_query(t.metadata, ${selectorTerm}::eql_v2_encrypted) as result + WHERE t.test_run_id = ${TEST_RUN_ID} + ` + + expect(rows.length).toBeGreaterThanOrEqual(1) + const matchingRow = rows.find((r) => r.id === id) + expect(matchingRow).toBeDefined() + await verifyRow(matchingRow!, plaintext) + }, 30000) + + it('finds row by deeply nested path ($.a.b.c)', async () => { + const plaintext = { a: { b: { c: 'deep-value' } }, marker: 'deep-path' } + const { id } = await insertRow(plaintext) + + const selectorTerm = await encryptQueryTerm('$.a.b.c', 'steVecSelector') + + const rows = await sql` + SELECT id, (metadata).data as metadata + FROM "protect-ci-jsonb" t, + eql_v2.jsonb_path_query(t.metadata, ${selectorTerm}::eql_v2_encrypted) as result + WHERE t.test_run_id = ${TEST_RUN_ID} + ` + + expect(rows.length).toBeGreaterThanOrEqual(1) + const matchingRow = rows.find((r) => r.id === id) + expect(matchingRow).toBeDefined() + await verifyRow(matchingRow!, plaintext) + }, 30000) + + it('non-matching path returns zero rows', async () => { + // Insert a doc that does NOT have $.nonexistent.path + const plaintext = { exists: true, marker: 'no-match-test' } + await insertRow(plaintext) + + const selectorTerm = await encryptQueryTerm( + '$.nonexistent.path', + 'steVecSelector', + ) + + const rows = await sql` + SELECT id, (metadata).data as metadata + FROM "protect-ci-jsonb" t, + eql_v2.jsonb_path_query(t.metadata, ${selectorTerm}::eql_v2_encrypted) as result + WHERE t.test_run_id = ${TEST_RUN_ID} + ` + + // No row should have this path + expect(rows.length).toBe(0) + }, 30000) + + it('multiple docs — only matching doc returned', async () => { + // Insert two docs: one with $.target.value, one without + const plaintextWithPath = { + target: { value: 'found-it' }, + marker: 'has-target', + } + const plaintextWithoutPath = { + other: { key: 'nope' }, + marker: 'no-target', + } + + const { id: idWith } = await insertRow(plaintextWithPath) + const { id: idWithout } = await insertRow(plaintextWithoutPath) + + const selectorTerm = await encryptQueryTerm( + '$.target.value', + 'steVecSelector', + ) + + const rows = await sql` + SELECT id, (metadata).data as metadata + FROM "protect-ci-jsonb" t, + eql_v2.jsonb_path_query(t.metadata, ${selectorTerm}::eql_v2_encrypted) as result + WHERE t.test_run_id = ${TEST_RUN_ID} + ` + + // The doc with $.target.value should be found + const matchingRow = rows.find((r) => r.id === idWith) + expect(matchingRow).toBeDefined() + + // The doc without $.target.value should NOT be found + const nonMatchingRow = rows.find((r) => r.id === idWithout) + expect(nonMatchingRow).toBeUndefined() + + // Decrypt and verify the matching row + await verifyRow(matchingRow!, plaintextWithPath) + }, 30000) + + it('finds row by simple top-level path (Simple)', async () => { + const plaintext = { role: 'path-tl-simple', extra: 'data' } + const { id } = await insertRow(plaintext) + + const selectorTerm = await encryptQueryTerm('$.role', 'steVecSelector') + + const rows = await sql.unsafe( + `SELECT id, (metadata).data as metadata + FROM "protect-ci-jsonb" t, + eql_v2.jsonb_path_query(t.metadata, '${selectorTerm}'::eql_v2_encrypted) as result + WHERE t.test_run_id = '${TEST_RUN_ID}'`, + ) + + expect(rows.length).toBeGreaterThanOrEqual(1) + const matchingRow = rows.find((r: any) => r.id === id) + expect(matchingRow).toBeDefined() + await verifyRow(matchingRow!, plaintext) + }, 30000) + + it('finds row by nested path (Simple)', async () => { + const plaintext = { + user: { email: 'nested-simple@test.com' }, + type: 'nested-path-simple', + } + const { id } = await insertRow(plaintext) + + const selectorTerm = await encryptQueryTerm( + '$.user.email', + 'steVecSelector', + ) + + const rows = await sql.unsafe( + `SELECT id, (metadata).data as metadata + FROM "protect-ci-jsonb" t, + eql_v2.jsonb_path_query(t.metadata, '${selectorTerm}'::eql_v2_encrypted) as result + WHERE t.test_run_id = '${TEST_RUN_ID}'`, + ) + + expect(rows.length).toBeGreaterThanOrEqual(1) + const matchingRow = rows.find((r: any) => r.id === id) + expect(matchingRow).toBeDefined() + await verifyRow(matchingRow!, plaintext) + }, 30000) + + it('finds with deep nested path (Simple)', async () => { + const plaintext = { + target: { nested: { value: 'deep-simple' } }, + marker: 'jpq-deep-simple', + } + const { id } = await insertRow(plaintext) + + const selectorTerm = await encryptQueryTerm( + '$.target.nested.value', + 'steVecSelector', + ) + + const rows = await sql.unsafe( + `SELECT id, (metadata).data as metadata + FROM "protect-ci-jsonb" t, + eql_v2.jsonb_path_query(t.metadata, '${selectorTerm}'::eql_v2_encrypted) as result + WHERE t.test_run_id = '${TEST_RUN_ID}'`, + ) + + expect(rows.length).toBeGreaterThanOrEqual(1) + const matchingRow = rows.find((r: any) => r.id === id) + expect(matchingRow).toBeDefined() + await verifyRow(matchingRow!, plaintext) + }, 30000) + + it('non-matching path returns zero rows (Simple)', async () => { + const plaintext = { data: true, marker: 'jpq-nomatch-simple' } + await insertRow(plaintext) + + const selectorTerm = await encryptQueryTerm( + '$.missing.path', + 'steVecSelector', + ) + + const rows = await sql.unsafe( + `SELECT id, (metadata).data as metadata + FROM "protect-ci-jsonb" t, + eql_v2.jsonb_path_query(t.metadata, '${selectorTerm}'::eql_v2_encrypted) as result + WHERE t.test_run_id = '${TEST_RUN_ID}'`, + ) + + expect(rows.length).toBe(0) + }, 30000) + }) + + // ─── Containment: @> term queries ────────────────────────────────── + + describe('containment: @> term queries', () => { + it('matches by key/value pair', async () => { + const plaintext = { role: 'admin-containment', department: 'engineering' } + const { id } = await insertRow(plaintext) + + const containmentTerm = await encryptQueryTerm( + { role: 'admin-containment' }, + 'steVecTerm', + ) + + const rows = await sql` + SELECT id, (metadata).data as metadata + FROM "protect-ci-jsonb" t + WHERE t.metadata @> ${containmentTerm}::eql_v2_encrypted + AND t.test_run_id = ${TEST_RUN_ID} + ` + + expect(rows.length).toBeGreaterThanOrEqual(1) + const matchingRow = rows.find((r) => r.id === id) + expect(matchingRow).toBeDefined() + await verifyRow(matchingRow!, plaintext) + }, 30000) + + it('matches by nested object structure', async () => { + const plaintext = { + user: { profile: { role: 'superadmin' } }, + active: true, + } + const { id } = await insertRow(plaintext) + + const containmentTerm = await encryptQueryTerm( + { user: { profile: { role: 'superadmin' } } }, + 'steVecTerm', + ) + + const rows = await sql` + SELECT id, (metadata).data as metadata + FROM "protect-ci-jsonb" t + WHERE t.metadata @> ${containmentTerm}::eql_v2_encrypted + AND t.test_run_id = ${TEST_RUN_ID} + ` + + expect(rows.length).toBeGreaterThanOrEqual(1) + const matchingRow = rows.find((r) => r.id === id) + expect(matchingRow).toBeDefined() + await verifyRow(matchingRow!, plaintext) + }, 30000) + + it('non-matching term returns zero rows', async () => { + const plaintext = { status: 'active', tier: 'free' } + await insertRow(plaintext) + + const containmentTerm = await encryptQueryTerm( + { status: 'nonexistent-value-xyz' }, + 'steVecTerm', + ) + + const rows = await sql` + SELECT id, (metadata).data as metadata + FROM "protect-ci-jsonb" t + WHERE t.metadata @> ${containmentTerm}::eql_v2_encrypted + AND t.test_run_id = ${TEST_RUN_ID} + ` + + expect(rows.length).toBe(0) + }, 30000) + }) + + // ─── Mixed and batch operations ──────────────────────────────────── + + describe('mixed and batch operations', () => { + it('batch encrypts selector + containment terms together', async () => { + const plaintext = { + user: { email: 'batch@test.com' }, + role: 'editor', + kind: 'batch-mixed', + } + const { id } = await insertRow(plaintext) + + const queryResult = await protectClient.encryptQuery([ + { + value: '$.user.email', + column: table.metadata, + table: table, + queryType: 'steVecSelector', + returnType: 'composite-literal', + }, + { + value: { role: 'editor' }, + column: table.metadata, + table: table, + queryType: 'steVecTerm', + returnType: 'composite-literal', + }, + ]) + + if (queryResult.failure) throw new Error(queryResult.failure.message) + const [selectorTerm, containmentTerm] = queryResult.data + + // Selector query: jsonb_path_query + const selectorRows = await sql` + SELECT id, (metadata).data as metadata + FROM "protect-ci-jsonb" t, + eql_v2.jsonb_path_query(t.metadata, ${selectorTerm}::eql_v2_encrypted) as result + WHERE t.test_run_id = ${TEST_RUN_ID} + ` + + expect(selectorRows.length).toBeGreaterThanOrEqual(1) + const selectorMatch = selectorRows.find((r) => r.id === id) + expect(selectorMatch).toBeDefined() + await verifyRow(selectorMatch!, plaintext) + + // Containment query: @> + const containmentRows = await sql` + SELECT id, (metadata).data as metadata + FROM "protect-ci-jsonb" + WHERE metadata @> ${containmentTerm}::eql_v2_encrypted + AND test_run_id = ${TEST_RUN_ID} + ` + + expect(containmentRows.length).toBeGreaterThanOrEqual(1) + const containmentMatch = containmentRows.find((r) => r.id === id) + expect(containmentMatch).toBeDefined() + await verifyRow(containmentMatch!, plaintext) + }, 30000) + + it('inferred vs explicit queryType produce same results', async () => { + const plaintext = { category: 'equivalence-test', priority: 'high' } + const { id } = await insertRow(plaintext) + + // Selector: inferred (searchableJson) vs explicit (steVecSelector) + const inferredSelectorTerm = await encryptQueryTerm( + '$.category', + 'searchableJson', + ) + const explicitSelectorTerm = await encryptQueryTerm( + '$.category', + 'steVecSelector', + ) + + const inferredRows = await sql` + SELECT id, (metadata).data as metadata + FROM "protect-ci-jsonb" t, + eql_v2.jsonb_path_query(t.metadata, ${inferredSelectorTerm}::eql_v2_encrypted) as result + WHERE t.test_run_id = ${TEST_RUN_ID} + ` + + const explicitRows = await sql` + SELECT id, (metadata).data as metadata + FROM "protect-ci-jsonb" t, + eql_v2.jsonb_path_query(t.metadata, ${explicitSelectorTerm}::eql_v2_encrypted) as result + WHERE t.test_run_id = ${TEST_RUN_ID} + ` + + expect(inferredRows.length).toBe(explicitRows.length) + expect(inferredRows.length).toBeGreaterThanOrEqual(1) + + // Both should find our inserted row + const inferredMatch = inferredRows.find((r) => r.id === id) + const explicitMatch = explicitRows.find((r) => r.id === id) + expect(inferredMatch).toBeDefined() + expect(explicitMatch).toBeDefined() + + // Decrypt and compare — both should yield identical plaintext + const inferredDecrypted = await protectClient.decryptModel({ + metadata: inferredMatch!.metadata, + }) + const explicitDecrypted = await protectClient.decryptModel({ + metadata: explicitMatch!.metadata, + }) + if (inferredDecrypted.failure) + throw new Error(inferredDecrypted.failure.message) + if (explicitDecrypted.failure) + throw new Error(explicitDecrypted.failure.message) + + expect(inferredDecrypted.data.metadata).toEqual( + explicitDecrypted.data.metadata, + ) + expect(inferredDecrypted.data.metadata).toEqual(plaintext) + + // Containment: inferred (searchableJson) vs explicit (steVecTerm) + const inferredContainmentTerm = await encryptQueryTerm( + { category: 'equivalence-test' }, + 'searchableJson', + ) + const explicitContainmentTerm = await encryptQueryTerm( + { category: 'equivalence-test' }, + 'steVecTerm', + ) + + const inferredTermRows = await sql` + SELECT id, (metadata).data as metadata + FROM "protect-ci-jsonb" + WHERE metadata @> ${inferredContainmentTerm}::eql_v2_encrypted + AND test_run_id = ${TEST_RUN_ID} + ` + + const explicitTermRows = await sql` + SELECT id, (metadata).data as metadata + FROM "protect-ci-jsonb" + WHERE metadata @> ${explicitContainmentTerm}::eql_v2_encrypted + AND test_run_id = ${TEST_RUN_ID} + ` + + expect(inferredTermRows.length).toBe(explicitTermRows.length) + expect(inferredTermRows.length).toBeGreaterThanOrEqual(1) + + const inferredTermMatch = inferredTermRows.find((r) => r.id === id) + const explicitTermMatch = explicitTermRows.find((r) => r.id === id) + expect(inferredTermMatch).toBeDefined() + expect(explicitTermMatch).toBeDefined() + + const inferredTermDecrypted = await protectClient.decryptModel({ + metadata: inferredTermMatch!.metadata, + }) + const explicitTermDecrypted = await protectClient.decryptModel({ + metadata: explicitTermMatch!.metadata, + }) + if (inferredTermDecrypted.failure) + throw new Error(inferredTermDecrypted.failure.message) + if (explicitTermDecrypted.failure) + throw new Error(explicitTermDecrypted.failure.message) + + expect(inferredTermDecrypted.data.metadata).toEqual( + explicitTermDecrypted.data.metadata, + ) + expect(inferredTermDecrypted.data.metadata).toEqual(plaintext) + }, 30000) + }) + + // ─── Escaped-composite-literal format ───────────────────────────── + + describe('escaped-composite-literal format', () => { + it('escaped selector → unwrap → query PG', async () => { + const plaintext = { + user: { email: 'escaped-sel@test.com' }, + marker: 'escaped-selector', + } + const { id } = await insertRow(plaintext) + + // Encrypt with both formats + const compositeData = await encryptQueryTerm( + '$.user.email', + 'steVecSelector', + 'composite-literal', + ) + const escapedData = (await encryptQueryTerm( + '$.user.email', + 'steVecSelector', + 'escaped-composite-literal', + )) as string + + // Verify escaped format and unwrap + expect(typeof escapedData).toBe('string') + expect(escapedData).toMatch(/^"\(.*\)"$/) + const unwrapped = JSON.parse(escapedData) + + expect(unwrapped).toBe(compositeData) + + // Use composite-literal form to query PG + const rows = await sql` + SELECT id, (metadata).data as metadata + FROM "protect-ci-jsonb" t, + eql_v2.jsonb_path_query(t.metadata, ${compositeData}::eql_v2_encrypted) as result + WHERE t.test_run_id = ${TEST_RUN_ID} + ` + + expect(rows.length).toBeGreaterThanOrEqual(1) + const matchingRow = rows.find((r) => r.id === id) + expect(matchingRow).toBeDefined() + await verifyRow(matchingRow!, plaintext) + }, 30000) + + it('escaped containment → unwrap → query PG', async () => { + const plaintext = { + role: 'escaped-containment-test', + department: 'security', + } + const { id } = await insertRow(plaintext) + + const escapedData = (await encryptQueryTerm( + { role: 'escaped-containment-test' }, + 'steVecTerm', + 'escaped-composite-literal', + )) as string + + // Verify escaped format and unwrap + expect(typeof escapedData).toBe('string') + expect(escapedData).toMatch(/^"\(.*\)"$/) + const unwrapped = JSON.parse(escapedData) + + // Unwrapped escaped format should be a valid composite-literal + expect(typeof unwrapped).toBe('string') + expect(unwrapped).toMatch(/^\(.*\)$/) + + // Use unwrapped composite-literal form to query PG + const rows = await sql` + SELECT id, (metadata).data as metadata + FROM "protect-ci-jsonb" + WHERE metadata @> ${unwrapped}::eql_v2_encrypted + AND test_run_id = ${TEST_RUN_ID} + ` + + expect(rows.length).toBeGreaterThanOrEqual(1) + const matchingRow = rows.find((r) => r.id === id) + expect(matchingRow).toBeDefined() + await verifyRow(matchingRow!, plaintext) + }, 30000) + + it('batch escaped format', async () => { + const plaintext = { + user: { email: 'batch-escaped@test.com' }, + role: 'batch-escaped-role', + marker: 'batch-escaped', + } + const { id } = await insertRow(plaintext) + + const queryResult = await protectClient.encryptQuery([ + { + value: '$.user.email', + column: table.metadata, + table: table, + queryType: 'steVecSelector', + returnType: 'escaped-composite-literal', + }, + { + value: { role: 'batch-escaped-role' }, + column: table.metadata, + table: table, + queryType: 'steVecTerm', + returnType: 'escaped-composite-literal', + }, + ]) + if (queryResult.failure) throw new Error(queryResult.failure.message) + + expect(queryResult.data).toHaveLength(2) + for (const item of queryResult.data) { + expect(typeof item).toBe('string') + expect(item).toMatch(/^"\(.*\)"$/) + } + + // Unwrap escaped format + const selectorUnwrapped = JSON.parse(queryResult.data[0] as string) + const containmentUnwrapped = JSON.parse(queryResult.data[1] as string) + + // Selector query + const selectorRows = await sql` + SELECT id, (metadata).data as metadata + FROM "protect-ci-jsonb" t, + eql_v2.jsonb_path_query(t.metadata, ${selectorUnwrapped}::eql_v2_encrypted) as result + WHERE t.test_run_id = ${TEST_RUN_ID} + ` + + expect(selectorRows.length).toBeGreaterThanOrEqual(1) + const selectorMatch = selectorRows.find((r) => r.id === id) + expect(selectorMatch).toBeDefined() + await verifyRow(selectorMatch!, plaintext) + + // Containment query + const containmentRows = await sql` + SELECT id, (metadata).data as metadata + FROM "protect-ci-jsonb" + WHERE metadata @> ${containmentUnwrapped}::eql_v2_encrypted + AND test_run_id = ${TEST_RUN_ID} + ` + + expect(containmentRows.length).toBeGreaterThanOrEqual(1) + const containmentMatch = containmentRows.find((r) => r.id === id) + expect(containmentMatch).toBeDefined() + await verifyRow(containmentMatch!, plaintext) + }, 30000) + }) + + // ─── LockContext integration ────────────────────────────────────── + + describe.skipIf(!userJwt)('LockContext integration', () => { + it('selector with LockContext', async () => { + const lc = new LockContext() + const lockContext = await lc.identify(userJwt!) + if (lockContext.failure) throw new Error(lockContext.failure.message) + + const plaintext = { + user: { email: 'lc-selector@test.com' }, + marker: 'lock-context-selector', + } + + const encrypted = await protectClient + .encryptModel({ metadata: plaintext }, table) + .withLockContext(lockContext.data) + if (encrypted.failure) throw new Error(encrypted.failure.message) + + const [inserted] = await sql` + INSERT INTO "protect-ci-jsonb" (metadata, test_run_id) + VALUES (${sql.json(encrypted.data.metadata)}::eql_v2_encrypted, ${TEST_RUN_ID}) + RETURNING id + ` + + const selectorResult = await protectClient + .encryptQuery('$.user.email', { + column: table.metadata, + table: table, + queryType: 'steVecSelector', + returnType: 'composite-literal', + }) + .withLockContext(lockContext.data) + .execute() + if (selectorResult.failure) + throw new Error(selectorResult.failure.message) + + const rows = await sql` + SELECT id, (metadata).data as metadata + FROM "protect-ci-jsonb" t, + eql_v2.jsonb_path_query(t.metadata, ${selectorResult.data}::eql_v2_encrypted) as result + WHERE t.test_run_id = ${TEST_RUN_ID} + ` + + expect(rows.length).toBeGreaterThanOrEqual(1) + const matchingRow = rows.find((r) => r.id === inserted.id) + expect(matchingRow).toBeDefined() + + const decrypted = await protectClient + .decryptModel({ metadata: matchingRow!.metadata }) + .withLockContext(lockContext.data) + if (decrypted.failure) throw new Error(decrypted.failure.message) + expect(decrypted.data.metadata).toEqual(plaintext) + }, 60000) + + it('containment with LockContext', async () => { + const lc = new LockContext() + const lockContext = await lc.identify(userJwt!) + if (lockContext.failure) throw new Error(lockContext.failure.message) + + const plaintext = { role: 'lc-containment-test', department: 'auth' } + + const encrypted = await protectClient + .encryptModel({ metadata: plaintext }, table) + .withLockContext(lockContext.data) + if (encrypted.failure) throw new Error(encrypted.failure.message) + + const [inserted] = await sql` + INSERT INTO "protect-ci-jsonb" (metadata, test_run_id) + VALUES (${sql.json(encrypted.data.metadata)}::eql_v2_encrypted, ${TEST_RUN_ID}) + RETURNING id + ` + + const containmentResult = await protectClient + .encryptQuery( + { role: 'lc-containment-test' }, + { + column: table.metadata, + table: table, + queryType: 'steVecTerm', + returnType: 'composite-literal', + }, + ) + .withLockContext(lockContext.data) + .execute() + if (containmentResult.failure) + throw new Error(containmentResult.failure.message) + + const rows = await sql` + SELECT id, (metadata).data as metadata + FROM "protect-ci-jsonb" + WHERE metadata @> ${containmentResult.data}::eql_v2_encrypted + AND test_run_id = ${TEST_RUN_ID} + ` + + expect(rows.length).toBeGreaterThanOrEqual(1) + const matchingRow = rows.find((r) => r.id === inserted.id) + expect(matchingRow).toBeDefined() + + const decrypted = await protectClient + .decryptModel({ metadata: matchingRow!.metadata }) + .withLockContext(lockContext.data) + if (decrypted.failure) throw new Error(decrypted.failure.message) + expect(decrypted.data.metadata).toEqual(plaintext) + }, 60000) + + it('batch with LockContext', async () => { + const lc = new LockContext() + const lockContext = await lc.identify(userJwt!) + if (lockContext.failure) throw new Error(lockContext.failure.message) + + const plaintext = { + user: { email: 'lc-batch@test.com' }, + role: 'lc-batch-role', + kind: 'lock-context-batch', + } + + const encrypted = await protectClient + .encryptModel({ metadata: plaintext }, table) + .withLockContext(lockContext.data) + if (encrypted.failure) throw new Error(encrypted.failure.message) + + const [inserted] = await sql` + INSERT INTO "protect-ci-jsonb" (metadata, test_run_id) + VALUES (${sql.json(encrypted.data.metadata)}::eql_v2_encrypted, ${TEST_RUN_ID}) + RETURNING id + ` + + const batchResult = await protectClient + .encryptQuery([ + { + value: '$.user.email', + column: table.metadata, + table: table, + queryType: 'steVecSelector', + returnType: 'composite-literal', + }, + { + value: { role: 'lc-batch-role' }, + column: table.metadata, + table: table, + queryType: 'steVecTerm', + returnType: 'composite-literal', + }, + ]) + .withLockContext(lockContext.data) + .execute() + if (batchResult.failure) throw new Error(batchResult.failure.message) + + const [selectorTerm, containmentTerm] = batchResult.data + + // Selector query + const selectorRows = await sql` + SELECT id, (metadata).data as metadata + FROM "protect-ci-jsonb" t, + eql_v2.jsonb_path_query(t.metadata, ${selectorTerm}::eql_v2_encrypted) as result + WHERE t.test_run_id = ${TEST_RUN_ID} + ` + + expect(selectorRows.length).toBeGreaterThanOrEqual(1) + const selectorMatch = selectorRows.find((r) => r.id === inserted.id) + expect(selectorMatch).toBeDefined() + + const selectorDecrypted = await protectClient + .decryptModel({ metadata: selectorMatch!.metadata }) + .withLockContext(lockContext.data) + if (selectorDecrypted.failure) + throw new Error(selectorDecrypted.failure.message) + expect(selectorDecrypted.data.metadata).toEqual(plaintext) + + // Containment query + const containmentRows = await sql` + SELECT id, (metadata).data as metadata + FROM "protect-ci-jsonb" + WHERE metadata @> ${containmentTerm}::eql_v2_encrypted + AND test_run_id = ${TEST_RUN_ID} + ` + + expect(containmentRows.length).toBeGreaterThanOrEqual(1) + const containmentMatch = containmentRows.find((r) => r.id === inserted.id) + expect(containmentMatch).toBeDefined() + + const containmentDecrypted = await protectClient + .decryptModel({ metadata: containmentMatch!.metadata }) + .withLockContext(lockContext.data) + if (containmentDecrypted.failure) + throw new Error(containmentDecrypted.failure.message) + expect(containmentDecrypted.data.metadata).toEqual(plaintext) + }, 60000) + }) + + // ─── Concurrent query operations ───────────────────────────────── + + describe('concurrent query operations', () => { + it('parallel selector queries', async () => { + // Insert 3 docs with distinct structures + const docs = [ + { alpha: { key: 'concurrent-sel-1' }, marker: 'concurrent-1' }, + { beta: { key: 'concurrent-sel-2' }, marker: 'concurrent-2' }, + { gamma: { key: 'concurrent-sel-3' }, marker: 'concurrent-3' }, + ] + + const insertedIds: number[] = [] + for (const plaintext of docs) { + const { id } = await insertRow(plaintext) + insertedIds.push(id) + } + + // Parallel encrypt 3 selector queries + const [q1, q2, q3] = await Promise.all([ + protectClient.encryptQuery('$.alpha.key', { + column: table.metadata, + table: table, + queryType: 'steVecSelector', + returnType: 'composite-literal', + }), + protectClient.encryptQuery('$.beta.key', { + column: table.metadata, + table: table, + queryType: 'steVecSelector', + returnType: 'composite-literal', + }), + protectClient.encryptQuery('$.gamma.key', { + column: table.metadata, + table: table, + queryType: 'steVecSelector', + returnType: 'composite-literal', + }), + ]) + + if (q1.failure) throw new Error(q1.failure.message) + if (q2.failure) throw new Error(q2.failure.message) + if (q3.failure) throw new Error(q3.failure.message) + + // Execute each against PG + const [rows1, rows2, rows3] = await Promise.all([ + sql` + SELECT id, (metadata).data as metadata FROM "protect-ci-jsonb" t, + eql_v2.jsonb_path_query(t.metadata, ${q1.data}::eql_v2_encrypted) as result + WHERE t.test_run_id = ${TEST_RUN_ID} + `, + sql` + SELECT id, (metadata).data as metadata FROM "protect-ci-jsonb" t, + eql_v2.jsonb_path_query(t.metadata, ${q2.data}::eql_v2_encrypted) as result + WHERE t.test_run_id = ${TEST_RUN_ID} + `, + sql` + SELECT id, (metadata).data as metadata FROM "protect-ci-jsonb" t, + eql_v2.jsonb_path_query(t.metadata, ${q3.data}::eql_v2_encrypted) as result + WHERE t.test_run_id = ${TEST_RUN_ID} + `, + ]) + + // Each query should find its respective doc and not others + expect(rows1.find((r) => r.id === insertedIds[0])).toBeDefined() + expect(rows1.find((r) => r.id === insertedIds[1])).toBeUndefined() + expect(rows1.find((r) => r.id === insertedIds[2])).toBeUndefined() + expect(rows2.find((r) => r.id === insertedIds[1])).toBeDefined() + expect(rows2.find((r) => r.id === insertedIds[0])).toBeUndefined() + expect(rows2.find((r) => r.id === insertedIds[2])).toBeUndefined() + expect(rows3.find((r) => r.id === insertedIds[2])).toBeDefined() + expect(rows3.find((r) => r.id === insertedIds[0])).toBeUndefined() + expect(rows3.find((r) => r.id === insertedIds[1])).toBeUndefined() + + // Decrypt and validate each matched row + const match1 = rows1.find((r) => r.id === insertedIds[0])! + const decrypted1 = await protectClient.decryptModel({ + metadata: match1.metadata, + }) + if (decrypted1.failure) throw new Error(decrypted1.failure.message) + expect(decrypted1.data.metadata).toEqual(docs[0]) + + const match2 = rows2.find((r) => r.id === insertedIds[1])! + const decrypted2 = await protectClient.decryptModel({ + metadata: match2.metadata, + }) + if (decrypted2.failure) throw new Error(decrypted2.failure.message) + expect(decrypted2.data.metadata).toEqual(docs[1]) + + const match3 = rows3.find((r) => r.id === insertedIds[2])! + const decrypted3 = await protectClient.decryptModel({ + metadata: match3.metadata, + }) + if (decrypted3.failure) throw new Error(decrypted3.failure.message) + expect(decrypted3.data.metadata).toEqual(docs[2]) + }, 60000) + + it('parallel containment queries', async () => { + const docs = [ + { role: 'concurrent-contain-1', tier: 'gold' }, + { role: 'concurrent-contain-2', tier: 'silver' }, + ] + + const insertedIds: number[] = [] + for (const plaintext of docs) { + const { id } = await insertRow(plaintext) + insertedIds.push(id) + } + + // Parallel encrypt 2 containment queries + const [c1, c2] = await Promise.all([ + protectClient.encryptQuery( + { role: 'concurrent-contain-1' }, + { + column: table.metadata, + table: table, + queryType: 'steVecTerm', + returnType: 'composite-literal', + }, + ), + protectClient.encryptQuery( + { role: 'concurrent-contain-2' }, + { + column: table.metadata, + table: table, + queryType: 'steVecTerm', + returnType: 'composite-literal', + }, + ), + ]) + + if (c1.failure) throw new Error(c1.failure.message) + if (c2.failure) throw new Error(c2.failure.message) + + const [rows1, rows2] = await Promise.all([ + sql` + SELECT id, (metadata).data as metadata FROM "protect-ci-jsonb" + WHERE metadata @> ${c1.data}::eql_v2_encrypted + AND test_run_id = ${TEST_RUN_ID} + `, + sql` + SELECT id, (metadata).data as metadata FROM "protect-ci-jsonb" + WHERE metadata @> ${c2.data}::eql_v2_encrypted + AND test_run_id = ${TEST_RUN_ID} + `, + ]) + + // Each finds only its target doc + expect(rows1.find((r) => r.id === insertedIds[0])).toBeDefined() + expect(rows1.find((r) => r.id === insertedIds[1])).toBeUndefined() + expect(rows2.find((r) => r.id === insertedIds[1])).toBeDefined() + expect(rows2.find((r) => r.id === insertedIds[0])).toBeUndefined() + + // Decrypt and validate each matched row + const match1 = rows1.find((r) => r.id === insertedIds[0])! + const decrypted1 = await protectClient.decryptModel({ + metadata: match1.metadata, + }) + if (decrypted1.failure) throw new Error(decrypted1.failure.message) + expect(decrypted1.data.metadata).toEqual(docs[0]) + + const match2 = rows2.find((r) => r.id === insertedIds[1])! + const decrypted2 = await protectClient.decryptModel({ + metadata: match2.metadata, + }) + if (decrypted2.failure) throw new Error(decrypted2.failure.message) + expect(decrypted2.data.metadata).toEqual(docs[1]) + }, 60000) + + it('parallel mixed encrypt+query', async () => { + const plaintext = { + user: { email: 'concurrent-mixed@test.com' }, + role: 'concurrent-mixed-role', + kind: 'mixed-concurrent', + } + + // Parallel: encryptModel + selector encryptQuery + containment encryptQuery + const [encryptedModel, selectorResult, containmentResult] = + await Promise.all([ + protectClient.encryptModel({ metadata: plaintext }, table), + protectClient.encryptQuery('$.user.email', { + column: table.metadata, + table: table, + queryType: 'steVecSelector', + returnType: 'composite-literal', + }), + protectClient.encryptQuery( + { role: 'concurrent-mixed-role' }, + { + column: table.metadata, + table: table, + queryType: 'steVecTerm', + returnType: 'composite-literal', + }, + ), + ]) + + if (encryptedModel.failure) + throw new Error(encryptedModel.failure.message) + if (selectorResult.failure) + throw new Error(selectorResult.failure.message) + if (containmentResult.failure) + throw new Error(containmentResult.failure.message) + + // Insert the encrypted doc + const [inserted] = await sql` + INSERT INTO "protect-ci-jsonb" (metadata, test_run_id) + VALUES (${sql.json(encryptedModel.data.metadata)}::eql_v2_encrypted, ${TEST_RUN_ID}) + RETURNING id + ` + + // Query with both terms + const [selectorRows, containmentRows] = await Promise.all([ + sql` + SELECT id, (metadata).data as metadata FROM "protect-ci-jsonb" t, + eql_v2.jsonb_path_query(t.metadata, ${selectorResult.data}::eql_v2_encrypted) as result + WHERE t.test_run_id = ${TEST_RUN_ID} + `, + sql` + SELECT id, (metadata).data as metadata FROM "protect-ci-jsonb" + WHERE metadata @> ${containmentResult.data}::eql_v2_encrypted + AND test_run_id = ${TEST_RUN_ID} + `, + ]) + + // Both should find the inserted row + expect(selectorRows.find((r) => r.id === inserted.id)).toBeDefined() + expect(containmentRows.find((r) => r.id === inserted.id)).toBeDefined() + // Verify result sets are bounded (not returning all rows) + expect(selectorRows.length).toBeGreaterThanOrEqual(1) + expect(containmentRows.length).toBeGreaterThanOrEqual(1) + + // Decrypt and validate both matched rows + const selectorMatch = selectorRows.find((r) => r.id === inserted.id)! + const selectorDecrypted = await protectClient.decryptModel({ + metadata: selectorMatch.metadata, + }) + if (selectorDecrypted.failure) + throw new Error(selectorDecrypted.failure.message) + expect(selectorDecrypted.data.metadata).toEqual(plaintext) + + const containmentMatch = containmentRows.find( + (r) => r.id === inserted.id, + )! + const containmentDecrypted = await protectClient.decryptModel({ + metadata: containmentMatch.metadata, + }) + if (containmentDecrypted.failure) + throw new Error(containmentDecrypted.failure.message) + expect(containmentDecrypted.data.metadata).toEqual(plaintext) + }, 60000) + }) + + // ─── eql (default) return type ────────────────────────────────────── + + describe('eql (default) return type', () => { + it('selector query using raw eql return type', async () => { + const plaintext = { + user: { email: 'eql-raw-sel@test.com' }, + marker: 'eql-raw-sel', + } + const { id } = await insertRow(plaintext) + + // Omit returnType — single-value encryptQuery returns raw Encrypted object + const queryResult = await protectClient.encryptQuery('$.user.email', { + column: table.metadata, + table: table, + queryType: 'steVecSelector', + }) + if (queryResult.failure) throw new Error(queryResult.failure.message) + const rawResult = queryResult.data + + // Must use sql.json() to pass raw Encrypted object to PG + const rows = await sql` + SELECT id, (metadata).data as metadata + FROM "protect-ci-jsonb" t, + eql_v2.jsonb_path_query(t.metadata, ${sql.json(rawResult)}::eql_v2_encrypted) as result + WHERE t.test_run_id = ${TEST_RUN_ID} + ` + + expect(rows.length).toBeGreaterThanOrEqual(1) + const matchingRow = rows.find((r) => r.id === id) + expect(matchingRow).toBeDefined() + await verifyRow(matchingRow!, plaintext) + }, 30000) + + it('containment query using raw eql return type', async () => { + const plaintext = { role: 'eql-raw-contain', marker: 'eql-raw-ct' } + const { id } = await insertRow(plaintext) + + // Omit returnType — single-value encryptQuery returns raw Encrypted object + const queryResult = await protectClient.encryptQuery( + { role: 'eql-raw-contain' }, + { + column: table.metadata, + table: table, + queryType: 'steVecTerm', + }, + ) + if (queryResult.failure) throw new Error(queryResult.failure.message) + const rawResult = queryResult.data + + // Must use sql.json() to pass raw Encrypted object to PG + const rows = await sql` + SELECT id, (metadata).data as metadata + FROM "protect-ci-jsonb" + WHERE metadata @> ${sql.json(rawResult)}::eql_v2_encrypted + AND test_run_id = ${TEST_RUN_ID} + ` + + expect(rows.length).toBeGreaterThanOrEqual(1) + const matchingRow = rows.find((r) => r.id === id) + expect(matchingRow).toBeDefined() + await verifyRow(matchingRow!, plaintext) + }, 30000) + }) + + // ─── Concurrent encrypt + decrypt stress ──────────────────────────── + + describe('concurrent encrypt + decrypt stress', () => { + it('concurrent encrypt + decrypt stress (10 parallel)', async () => { + const docs = Array.from({ length: 10 }, (_, i) => ({ + user: { email: `stress-${i}@test.com` }, + role: `stress-role-${i}`, + index: i, + marker: `stress-${i}`, + })) + + // Insert all 10 docs + const insertedIds: number[] = [] + for (const plaintext of docs) { + const { id } = await insertRow(plaintext) + insertedIds.push(id) + } + + // 10 parallel encrypt-query-decrypt pipelines + const results = await Promise.all( + docs.map(async (plaintext, i) => { + // Encrypt a selector query + const selectorTerm = await encryptQueryTerm( + '$.user.email', + 'steVecSelector', + ) + + // Query PG + const rows = await sql` + SELECT id, (metadata).data as metadata + FROM "protect-ci-jsonb" t, + eql_v2.jsonb_path_query(t.metadata, ${selectorTerm}::eql_v2_encrypted) as result + WHERE t.id = ${insertedIds[i]} + ` + + expect(rows).toHaveLength(1) + + // Decrypt + const decrypted = await protectClient.decryptModel({ + metadata: rows[0].metadata, + }) + if (decrypted.failure) throw new Error(decrypted.failure.message) + + return decrypted.data.metadata + }), + ) + + // Assert all 10 return correct plaintext + expect(results).toHaveLength(10) + results.forEach((result, i) => { + expect(result).toEqual(docs[i]) + }) + }, 120000) + }) +}) diff --git a/packages/stack/package.json b/packages/stack/package.json index e48bf7a0..c5a9e732 100644 --- a/packages/stack/package.json +++ b/packages/stack/package.json @@ -169,6 +169,7 @@ "drizzle-orm": ">=0.33", "execa": "^9.5.2", "json-schema-to-typescript": "^15.0.2", + "postgres": "^3.4.8", "tsup": "catalog:repo", "tsx": "catalog:repo", "typescript": "catalog:repo", diff --git a/packages/stack/src/drizzle/index.ts b/packages/stack/src/drizzle/index.ts index 3b2d2d31..02d3adee 100644 --- a/packages/stack/src/drizzle/index.ts +++ b/packages/stack/src/drizzle/index.ts @@ -23,6 +23,11 @@ export type EncryptedColumnConfig = { * Enable order and range index for sorting and range queries. */ orderAndRange?: boolean + /** + * Enable searchable JSON index for JSONB path queries. + * Requires dataType: 'json'. + */ + searchableJson?: boolean } /** @@ -53,6 +58,7 @@ const columnConfigMap = new Map< * - `freeTextSearch`: Enables free text search index. Can be a boolean for default options, or an object for custom configuration. * - `equality`: Enables equality index. Can be a boolean for default options, or an array of token filters. * - `orderAndRange`: Enables order and range index for sorting and range queries. + * - `searchableJson`: Enables searchable JSON index for JSONB path queries on encrypted JSON columns. * * See {@link EncryptedColumnConfig}. * @@ -197,4 +203,8 @@ export { extractEncryptionSchema } from './schema-extraction.js' * passing them to Drizzle, enabling searchable encryption in standard * Drizzle queries. */ -export { createEncryptionOperators } from './operators.js' +export { + createEncryptionOperators, + EncryptionOperatorError, + EncryptionConfigError, +} from './operators.js' diff --git a/packages/stack/src/drizzle/operators.ts b/packages/stack/src/drizzle/operators.ts index 64a9523a..008c6f9f 100644 --- a/packages/stack/src/drizzle/operators.ts +++ b/packages/stack/src/drizzle/operators.ts @@ -1,6 +1,6 @@ import type { EncryptionClient } from '@/encryption/ffi' import type { ProtectColumn, ProtectTable, ProtectTableColumn } from '@/schema' -import type { QueryTypeName } from '@/types' +import { type QueryTypeName, queryTypes } from '@/types' import { type SQL, type SQLWrapper, @@ -254,6 +254,7 @@ interface ValueToEncrypt { readonly column: SQLWrapper readonly columnInfo: ColumnInfo readonly queryType?: QueryTypeName + readonly originalIndex: number } /** @@ -298,6 +299,7 @@ async function encryptValues( column, columnInfo, queryType, + originalIndex: i, }) } @@ -311,6 +313,7 @@ async function encryptValues( { column: ProtectColumn table: ProtectTable + columnName: string values: Array<{ value: string | number index: number @@ -321,7 +324,12 @@ async function encryptValues( >() let valueIndex = 0 - for (const { value, column, columnInfo, queryType } of valuesToEncrypt) { + for (const { + value, + columnInfo, + queryType, + originalIndex, + } of valuesToEncrypt) { // Safe access with validation - we know these exist from earlier checks if ( !columnInfo.config || @@ -332,32 +340,25 @@ async function encryptValues( } const columnName = columnInfo.config.name - let group = columnGroups.get(columnName) + const groupKey = `${columnInfo.tableName ?? 'unknown'}/${columnName}` + let group = columnGroups.get(groupKey) if (!group) { group = { column: columnInfo.encryptedColumn, table: columnInfo.encryptedTable, + columnName, values: [], resultIndices: [], } - columnGroups.set(columnName, group) + columnGroups.set(groupKey, group) } group.values.push({ value, index: valueIndex++, queryType }) - - // Find the original index in the results array - const originalIndex = values.findIndex( - (v, idx) => - v.column === column && - toPlaintext(v.value) === value && - results[idx] === undefined, - ) - if (originalIndex >= 0) { - group.resultIndices.push(originalIndex) - } + group.resultIndices.push(originalIndex) } // Encrypt all values for each column in batches - for (const [columnName, group] of columnGroups) { + for (const [, group] of columnGroups) { + const { columnName } = group try { const terms = group.values.map((v) => ({ value: v.value, @@ -701,7 +702,7 @@ function createComparisonOperator( tableCache, undefined, // min undefined, // max - 'orderAndRange', + queryTypes.orderAndRange, ) as Promise } @@ -735,7 +736,7 @@ function createComparisonOperator( tableCache, undefined, // min undefined, // max - 'equality', + queryTypes.equality, ) as Promise } @@ -799,7 +800,7 @@ function createRangeOperator( tableCache, min, max, - 'orderAndRange', + queryTypes.orderAndRange, ) as Promise } @@ -858,7 +859,75 @@ function createTextSearchOperator( tableCache, undefined, // min undefined, // max - 'freeTextSearch', + queryTypes.freeTextSearch, + ) as Promise +} + +/** + * Creates a JSONB operator that encrypts a JSON path selector and wraps it + * in the appropriate `eql_v2` function call. + * + * Supports `jsonbPathQueryFirst`, `jsonbGet`, and `jsonbPathExists`. + * The column must have `searchableJson` enabled in its {@link EncryptedColumnConfig}. + */ +function createJsonbOperator( + operator: 'jsonbPathQueryFirst' | 'jsonbGet' | 'jsonbPathExists', + left: SQLWrapper, + right: unknown, + columnInfo: ColumnInfo, + encryptionClient: EncryptionClient, + defaultTable: ProtectTable | undefined, + tableCache: Map>, +): Promise { + const { config } = columnInfo + const encryptedSelector = (value: unknown) => + sql`${bindIfParam(value, left)}::eql_v2_encrypted` + + if (!config?.searchableJson) { + throw new EncryptionOperatorError( + `The ${operator} operator requires searchableJson to be enabled on the column configuration.`, + { + columnName: columnInfo.columnName, + tableName: columnInfo.tableName, + operator, + }, + ) + } + + const executeFn = (encrypted: unknown) => { + if (encrypted === undefined) { + throw new EncryptionOperatorError( + `Encryption failed for ${operator} operator`, + { + columnName: columnInfo.columnName, + tableName: columnInfo.tableName, + operator, + }, + ) + } + switch (operator) { + case 'jsonbPathQueryFirst': + return sql`eql_v2.jsonb_path_query_first(${left}, ${encryptedSelector(encrypted)})` + case 'jsonbGet': + return sql`${left} -> ${encryptedSelector(encrypted)}` + case 'jsonbPathExists': + return sql`eql_v2.jsonb_path_exists(${left}, ${encryptedSelector(encrypted)})` + } + } + + return createLazyOperator( + operator, + left, + right, + executeFn, + true, + columnInfo, + encryptionClient, + defaultTable, + tableCache, + undefined, + undefined, + queryTypes.steVecSelector, ) as Promise } @@ -1043,6 +1112,39 @@ export function createEncryptionOperators(encryptionClient: EncryptionClient): { */ ilike: (left: SQLWrapper, right: unknown) => Promise | SQL notIlike: (left: SQLWrapper, right: unknown) => Promise | SQL + + /** + * JSONB path query first operator for encrypted columns with searchable JSON. + * Requires `searchableJson` to be set on {@link EncryptedColumnConfig}. + * + * Encrypts the JSON path selector and calls `eql_v2.jsonb_path_query_first()`, + * casting the parameter to `eql_v2_encrypted`. + * + * @throws {EncryptionOperatorError} If the column does not have `searchableJson` enabled. + */ + jsonbPathQueryFirst: (left: SQLWrapper, right: unknown) => Promise + + /** + * JSONB get operator for encrypted columns with searchable JSON. + * Requires `searchableJson` to be set on {@link EncryptedColumnConfig}. + * + * Encrypts the JSON path selector and uses the `->` operator, + * casting the parameter to `eql_v2_encrypted`. + * + * @throws {EncryptionOperatorError} If the column does not have `searchableJson` enabled. + */ + jsonbGet: (left: SQLWrapper, right: unknown) => Promise + + /** + * JSONB path exists operator for encrypted columns with searchable JSON. + * Requires `searchableJson` to be set on {@link EncryptedColumnConfig}. + * + * Encrypts the JSON path selector and calls `eql_v2.jsonb_path_exists()`, + * casting the parameter to `eql_v2_encrypted`. + * + * @throws {EncryptionOperatorError} If the column does not have `searchableJson` enabled. + */ + jsonbPathExists: (left: SQLWrapper, right: unknown) => Promise // Array operators inArray: (left: SQLWrapper, right: unknown[] | SQLWrapper) => Promise notInArray: (left: SQLWrapper, right: unknown[] | SQLWrapper) => Promise @@ -1283,6 +1385,66 @@ export function createEncryptionOperators(encryptionClient: EncryptionClient): { ) } + /** + * JSONB path query first operator - encrypts the selector and calls + * `eql_v2.jsonb_path_query_first()` for encrypted columns with searchable JSON. + */ + const encryptedJsonbPathQueryFirst = ( + left: SQLWrapper, + right: unknown, + ): Promise => { + const columnInfo = getColumnInfo(left, defaultTable, tableCache) + return createJsonbOperator( + 'jsonbPathQueryFirst', + left, + right, + columnInfo, + encryptionClient, + defaultTable, + tableCache, + ) + } + + /** + * JSONB get operator - encrypts the selector and uses the `->` operator + * for encrypted columns with searchable JSON. + */ + const encryptedJsonbGet = ( + left: SQLWrapper, + right: unknown, + ): Promise => { + const columnInfo = getColumnInfo(left, defaultTable, tableCache) + return createJsonbOperator( + 'jsonbGet', + left, + right, + columnInfo, + encryptionClient, + defaultTable, + tableCache, + ) + } + + /** + * JSONB path exists operator - encrypts the selector and calls + * `eql_v2.jsonb_path_exists()` for encrypted columns with searchable JSON. + */ + const encryptedJsonbPathExists = ( + left: SQLWrapper, + right: unknown, + ): Promise => { + const columnInfo = getColumnInfo(left, defaultTable, tableCache) + return createJsonbOperator( + 'jsonbPathExists', + left, + right, + columnInfo, + encryptionClient, + defaultTable, + tableCache, + ) + } + /** * In array operator - encrypts all values in the array */ @@ -1307,7 +1469,7 @@ export function createEncryptionOperators(encryptionClient: EncryptionClient): { right.map((value) => ({ value, column: left, - queryType: 'equality' as const, + queryType: queryTypes.equality, })), defaultTable, tableCache, @@ -1353,7 +1515,7 @@ export function createEncryptionOperators(encryptionClient: EncryptionClient): { right.map((value) => ({ value, column: left, - queryType: 'equality' as const, + queryType: queryTypes.equality, })), defaultTable, tableCache, @@ -1735,6 +1897,11 @@ export function createEncryptionOperators(encryptionClient: EncryptionClient): { ilike: encryptedIlike, notIlike: encryptedNotIlike, + // Searchable JSON operators + jsonbPathQueryFirst: encryptedJsonbPathQueryFirst, + jsonbGet: encryptedJsonbGet, + jsonbPathExists: encryptedJsonbPathExists, + // Array operators inArray: encryptedInArray, notInArray: encryptedNotInArray, diff --git a/packages/stack/src/drizzle/schema-extraction.ts b/packages/stack/src/drizzle/schema-extraction.ts index b98386a0..6d5915ab 100644 --- a/packages/stack/src/drizzle/schema-extraction.ts +++ b/packages/stack/src/drizzle/schema-extraction.ts @@ -87,6 +87,15 @@ export function extractEncryptionSchema>( } } + if (config.searchableJson) { + if (config.dataType !== 'json') { + throw new Error( + `Column "${columnName}" has searchableJson enabled but dataType is "${config.dataType ?? 'string'}". searchableJson requires dataType: 'json'.`, + ) + } + csCol.searchableJson() + } + columns[actualColumnName] = csCol } } diff --git a/packages/stack/src/encryption/ffi/operations/batch-encrypt-query.ts b/packages/stack/src/encryption/ffi/operations/batch-encrypt-query.ts index 1f535ff7..1382ea31 100644 --- a/packages/stack/src/encryption/ffi/operations/batch-encrypt-query.ts +++ b/packages/stack/src/encryption/ffi/operations/batch-encrypt-query.ts @@ -1,8 +1,5 @@ import { getErrorCode } from '@/encryption/ffi/helpers/error-code' -import { - encryptedToCompositeLiteral, - encryptedToEscapedCompositeLiteral, -} from '@/encryption/helpers' +import { formatEncryptedResult } from '@/encryption/helpers' import { type EncryptionError, EncryptionErrorTypes } from '@/errors' import type { Context, LockContext } from '@/identity' import type { Client, EncryptedQueryResult, ScalarQueryTerm } from '@/types' @@ -95,13 +92,7 @@ function assembleResults( nonNullTerms.forEach(({ term, originalIndex }, i) => { const encrypted = encryptedValues[i] - if (term.returnType === 'composite-literal') { - results[originalIndex] = encryptedToCompositeLiteral(encrypted) - } else if (term.returnType === 'escaped-composite-literal') { - results[originalIndex] = encryptedToEscapedCompositeLiteral(encrypted) - } else { - results[originalIndex] = encrypted - } + results[originalIndex] = formatEncryptedResult(encrypted, term.returnType) }) return results diff --git a/packages/stack/src/encryption/ffi/operations/encrypt-query.ts b/packages/stack/src/encryption/ffi/operations/encrypt-query.ts index 2386eec2..638fcef5 100644 --- a/packages/stack/src/encryption/ffi/operations/encrypt-query.ts +++ b/packages/stack/src/encryption/ffi/operations/encrypt-query.ts @@ -1,7 +1,8 @@ import { getErrorCode } from '@/encryption/ffi/helpers/error-code' +import { formatEncryptedResult } from '@/encryption/helpers' import { type EncryptionError, EncryptionErrorTypes } from '@/errors' import type { LockContext } from '@/identity' -import type { Client, EncryptQueryOptions, Encrypted } from '@/types' +import type { Client, EncryptQueryOptions, EncryptedQueryResult } from '@/types' import { createRequestLogger } from '@/utils/logger' import { type Result, withResult } from '@byteslice/result' import { @@ -19,7 +20,7 @@ import { EncryptionOperation } from './base-operation' /** * @internal Use {@link EncryptionClient.encryptQuery} instead. */ -export class EncryptQueryOperation extends EncryptionOperation { +export class EncryptQueryOperation extends EncryptionOperation { constructor( private client: Client, private plaintext: JsPlaintext | null, @@ -40,7 +41,7 @@ export class EncryptQueryOperation extends EncryptionOperation { ) } - public async execute(): Promise> { + public async execute(): Promise> { const log = createRequestLogger() log.set({ op: 'encryptQuery', @@ -80,7 +81,7 @@ export class EncryptQueryOperation extends EncryptionOperation { this.opts.column.getName(), ) - return await ffiEncryptQuery(this.client, { + const encrypted = await ffiEncryptQuery(this.client, { plaintext: this.plaintext as JsPlaintext, column: this.opts.column.getName(), table: this.opts.table.tableName, @@ -88,6 +89,8 @@ export class EncryptQueryOperation extends EncryptionOperation { queryOp, unverifiedContext: metadata, }) + + return formatEncryptedResult(encrypted, this.opts.returnType) }, (error: unknown) => { log.set({ errorCode: getErrorCode(error) ?? 'unknown' }) @@ -110,7 +113,7 @@ export class EncryptQueryOperation extends EncryptionOperation { /** * @internal Use {@link EncryptionClient.encryptQuery} with `.withLockContext()` instead. */ -export class EncryptQueryOperationWithLockContext extends EncryptionOperation { +export class EncryptQueryOperationWithLockContext extends EncryptionOperation { constructor( private client: Client, private plaintext: JsPlaintext | null, @@ -122,7 +125,7 @@ export class EncryptQueryOperationWithLockContext extends EncryptionOperation> { + public async execute(): Promise> { const log = createRequestLogger() log.set({ op: 'encryptQuery', @@ -170,7 +173,7 @@ export class EncryptQueryOperationWithLockContext extends EncryptionOperation { log.set({ errorCode: getErrorCode(error) ?? 'unknown' }) diff --git a/packages/stack/src/encryption/helpers/index.ts b/packages/stack/src/encryption/helpers/index.ts index 499cec66..1c45d26c 100644 --- a/packages/stack/src/encryption/helpers/index.ts +++ b/packages/stack/src/encryption/helpers/index.ts @@ -1,4 +1,4 @@ -import type { Encrypted, KeysetIdentifier } from '@/types' +import type { Encrypted, EncryptedQueryResult, KeysetIdentifier } from '@/types' import type { Encrypted as CipherStashEncrypted, KeysetIdentifier as KeysetIdentifierFfi, @@ -53,6 +53,26 @@ export function encryptedToEscapedCompositeLiteral( return JSON.stringify(encryptedToCompositeLiteral(obj)) } +/** + * Format an encrypted result based on the requested return type. + * + * - `'composite-literal'` → PostgreSQL composite literal string `("json")` + * - `'escaped-composite-literal'` → escaped variant `"(\"json\")"` + * - default (`'eql'` or omitted) → raw encrypted object + */ +export function formatEncryptedResult( + encrypted: CipherStashEncrypted, + returnType?: string, +): EncryptedQueryResult { + if (returnType === 'composite-literal') { + return encryptedToCompositeLiteral(encrypted) + } + if (returnType === 'escaped-composite-literal') { + return encryptedToEscapedCompositeLiteral(encrypted) + } + return encrypted +} + /** * Helper function to transform a model's encrypted fields into PostgreSQL composite types */ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index be8d8361..6d6f77e4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -634,6 +634,9 @@ importers: json-schema-to-typescript: specifier: ^15.0.2 version: 15.0.4 + postgres: + specifier: ^3.4.7 + version: 3.4.7 tsup: specifier: catalog:repo version: 8.4.0(jiti@2.6.1)(postcss@8.5.6)(tsx@4.19.3)(typescript@5.6.3) diff --git a/skills/stash-drizzle/SKILL.md b/skills/stash-drizzle/SKILL.md index 724d8b6c..5cd9703a 100644 --- a/skills/stash-drizzle/SKILL.md +++ b/skills/stash-drizzle/SKILL.md @@ -81,9 +81,10 @@ const usersTable = pgTable("users", { orderAndRange: true, }), - // Encrypted JSON object + // Encrypted JSON object with searchable JSONB queries profile: encryptedType<{ name: string; bio: string }>("profile", { dataType: "json", + searchableJson: true, }), // Non-encrypted columns @@ -100,6 +101,7 @@ const usersTable = pgTable("users", { | `equality` | `boolean` \| `TokenFilter[]` | Enable equality index | | `freeTextSearch` | `boolean` \| `MatchIndexOpts` | Enable free-text search index | | `orderAndRange` | `boolean` | Enable ORE index for sorting and range queries | +| `searchableJson` | `boolean` | Enable JSONB path queries (requires `dataType: "json"`) | The generic type parameter `` sets the TypeScript type for the decrypted value. @@ -234,6 +236,50 @@ const results = await db .orderBy(encryptionOps.desc(usersTable.age)) ``` +## JSONB Queries + +Query encrypted JSON columns using JSONB operators. These require `searchableJson: true` and `dataType: "json"` in the column's `encryptedType` config. + +### Check path existence + +```typescript +// Check if a JSONB path exists in an encrypted column +const results = await db + .select() + .from(usersTable) + .where(await encryptionOps.jsonbPathExists(usersTable.profile, "$.bio")) +``` + +### Extract value at path + +```typescript +// Extract the first matching value at a JSONB path +const result = await encryptionOps.jsonbPathQueryFirst(usersTable.profile, "$.name") +``` + +### Get value with `->` operator + +```typescript +// Get a value using the JSONB -> operator +const result = await encryptionOps.jsonbGet(usersTable.profile, "$.name") +``` + +> **Note:** `jsonbPathExists` returns a boolean and can be used in `WHERE` clauses. `jsonbPathQueryFirst` and `jsonbGet` return encrypted values — use them in `SELECT` expressions. + +### Combine JSONB with other operators + +```typescript +const results = await db + .select() + .from(usersTable) + .where( + await encryptionOps.and( + encryptionOps.jsonbPathExists(usersTable.profile, "$.name"), + encryptionOps.eq(usersTable.email, "jane@example.com"), + ), + ) +``` + ## Batched Conditions (and / or) Use `encryptionOps.and()` and `encryptionOps.or()` to batch multiple encrypted conditions into a single ZeroKMS call. This is more efficient than awaiting each operator individually. @@ -303,6 +349,9 @@ if (!decrypted.failure) { | `notIlike(col, pattern)` | NOT ILIKE | `.freeTextSearch()` | | `inArray(col, values)` | IN array | `.equality()` | | `notInArray(col, values)` | NOT IN array | `.equality()` | +| `jsonbPathQueryFirst(col, selector)` | Extract first value at JSONB path | `.searchableJson()` | +| `jsonbGet(col, selector)` | Get value using JSONB `->` operator | `.searchableJson()` | +| `jsonbPathExists(col, selector)` | Check if JSONB path exists | `.searchableJson()` | ### Sort Operators (sync) diff --git a/skills/stash-encryption/SKILL.md b/skills/stash-encryption/SKILL.md index fc7fa3af..95750752 100644 --- a/skills/stash-encryption/SKILL.md +++ b/skills/stash-encryption/SKILL.md @@ -61,7 +61,38 @@ If `config` is omitted, the client reads `CS_*` environment variables automatica ### Logging ```bash -PROTECT_LOG_LEVEL=debug # debug | info | error +STASH_LOG_LEVEL=debug # debug | info | warn | error (default: info) +``` + +#### Programmatic Logging Configuration + +```typescript +const client = await Encryption({ + schemas: [users], + logging: { + enabled: true, // toggle logging on/off (default: true) + pretty: true, // pretty-print in development (default: auto-detected) + }, +}) +``` + +#### Log Draining + +Send structured logs to an external observability platform: + +```typescript +const client = await Encryption({ + schemas: [users], + logging: { + drain: (ctx) => { + // Forward to Axiom, Datadog, OTLP, etc. + fetch("https://your-service.com/logs", { + method: "POST", + body: JSON.stringify(ctx.event), + }) + }, + }, +}) ``` The SDK never logs plaintext data. @@ -74,6 +105,9 @@ The SDK never logs plaintext data. | `@cipherstash/stack/schema` | `encryptedTable`, `encryptedColumn`, `encryptedValue`, schema types | | `@cipherstash/stack/identity` | `LockContext` class and identity types | | `@cipherstash/stack/secrets` | `Secrets` class and secrets types | +| `@cipherstash/stack/drizzle` | `encryptedType`, `extractEncryptionSchema`, `createEncryptionOperators` for Drizzle ORM | +| `@cipherstash/stack/supabase` | `encryptedSupabase` wrapper for Supabase | +| `@cipherstash/stack/dynamodb` | `encryptedDynamoDB` helper for DynamoDB | | `@cipherstash/stack/client` | Client-safe exports (schema builders + types only, no native FFI) | | `@cipherstash/stack/types` | All TypeScript types | @@ -111,7 +145,7 @@ const documents = encryptedTable("documents", { | `.equality()` | Exact match lookups | `'equality'` | | `.freeTextSearch(opts?)` | Full-text / fuzzy search | `'freeTextSearch'` | | `.orderAndRange()` | Sorting, comparison, range queries | `'orderAndRange'` | -| `.searchableJson()` | Encrypted JSONB path and containment queries | `'searchableJson'` | +| `.searchableJson()` | Encrypted JSONB path and containment queries (auto-sets `dataType` to `'json'`) | `'searchableJson'` | | `.dataType(cast)` | Set plaintext data type | N/A | **Supported data types:** `'string'` (default), `'number'`, `'boolean'`, `'date'`, `'bigint'`, `'json'` @@ -148,17 +182,19 @@ type UserEncrypted = InferEncrypted import { Encryption } from "@cipherstash/stack" const client = await Encryption({ schemas: [users, documents] }) +``` -if (client.failure) { - console.error("Init failed:", client.failure.message) - // client.failure.type === "ClientInitError" -} else { - // client.data is the EncryptionClient +The `Encryption()` function returns `Promise` and throws on error (e.g., bad credentials, missing config, invalid keyset UUID). At least one schema is required. + +```typescript +// Error handling +try { + const client = await Encryption({ schemas: [users] }) +} catch (error) { + console.error("Init failed:", error.message) } ``` -At least one schema is required. The `Encryption()` function returns a `Result`. - ## Encrypt and Decrypt Single Values ```typescript @@ -285,6 +321,29 @@ const rangeQuery = await client.encryptQuery(25, { If `queryType` is omitted, it's auto-inferred from the column's configured indexes (priority: unique > match > ore > ste_vec). +### Query Result Formatting (`returnType`) + +By default `encryptQuery` returns an `Encrypted` object (the raw EQL JSON payload). Use `returnType` to change the output format: + +| `returnType` | Output | Use case | +|---|---|---| +| `'eql'` (default) | `Encrypted` object | Parameterized queries, ORMs accepting JSON | +| `'composite-literal'` | `string` | Supabase `.eq()`, string-based APIs | +| `'escaped-composite-literal'` | `string` | Embedding inside another string or JSON value | + +```typescript +// Get a composite literal string for use with Supabase +const term = await client.encryptQuery("alice@example.com", { + column: users.email, + table: users, + queryType: "equality", + returnType: "composite-literal", +}) +// term.data is a string +``` + +Each term in a batch can have its own `returnType`. + ### Searchable JSON For columns using `.searchableJson()`, the query type is auto-inferred from the plaintext: @@ -463,8 +522,8 @@ All method signatures on the encryption client remain the same. The `Result` pat |---|---|---| | `encrypt` | `(plaintext, { column, table })` | `EncryptOperation` | | `decrypt` | `(encryptedData)` | `DecryptOperation` | -| `encryptQuery` | `(plaintext, { column, table, queryType? })` | `EncryptQueryOperation` | -| `encryptQuery` | `(terms: ScalarQueryTerm[])` | `BatchEncryptQueryOperation` | +| `encryptQuery` | `(plaintext, { column, table, queryType?, returnType? })` | `EncryptQueryOperation` | +| `encryptQuery` | `(terms: readonly ScalarQueryTerm[])` | `BatchEncryptQueryOperation` | | `encryptModel` | `(model, table)` | `EncryptModelOperation` | | `decryptModel` | `(encryptedModel)` | `DecryptModelOperation` | | `bulkEncrypt` | `(plaintexts, { column, table })` | `BulkEncryptOperation` | @@ -477,7 +536,7 @@ All operations are thenable (awaitable) and support `.withLockContext()` and `.a ### Schema Builders ```typescript -encryptedTable(tableName: string, columns: Record) +encryptedTable(tableName: string, columns: Record) encryptedColumn(columnName: string) // chainable: .equality(), .freeTextSearch(), .orderAndRange(), .searchableJson(), .dataType() -encryptedValue(valueName: string) // for nested encrypted values +encryptedValue(valueName: string) // for nested encrypted values, chainable: .dataType() ``` diff --git a/skills/stash-secrets/SKILL.md b/skills/stash-secrets/SKILL.md index af1f7319..d862a1e8 100644 --- a/skills/stash-secrets/SKILL.md +++ b/skills/stash-secrets/SKILL.md @@ -62,6 +62,8 @@ const result = await secrets.set("DATABASE_URL", "postgres://user:pass@host:5432 if (result.failure) { console.error("Failed:", result.failure.message) // result.failure.type: "ApiError" | "NetworkError" | "ClientError" | "EncryptionError" +} else { + console.log(result.data.message) // success message } ``` @@ -91,6 +93,8 @@ if (!result.failure) { } ``` +**Constraints:** `getMany` requires a minimum of 2 secret names and a maximum of 100 names per request. + **Use `getMany` over multiple `get` calls** - it's significantly more efficient because it batches the decryption into a single ZeroKMS operation. ### List Secret Names @@ -180,11 +184,11 @@ interface SecretsConfig { ```typescript interface SecretMetadata { - id?: string + id: string name: string environment: string - createdAt?: string - updatedAt?: string + createdAt: string + updatedAt: string } ``` @@ -205,11 +209,11 @@ All operations return `Result` with either `data` or `failure`. | Method | Signature | Returns | |---|---|---| -| `set` | `(name: string, value: string)` | `Promise>` | +| `set` | `(name: string, value: string)` | `Promise>` | | `get` | `(name: string)` | `Promise>` | -| `getMany` | `(names: string[])` | `Promise, SecretsError>>` | +| `getMany` | `(names: string[])` (min 2, max 100) | `Promise, SecretsError>>` | | `list` | `()` | `Promise>` | -| `delete` | `(name: string)` | `Promise>` | +| `delete` | `(name: string)` | `Promise>` | ## Patterns