Update JSON functions

Author: simolus3

packages/sync-rules/src/SqlBucketDescriptor.ts

@@ -64,7 +64,7 @@ export class SqlBucketDescriptor implements BucketSource {
     if (this.bucketParameters == null) {
       throw new Error('Bucket parameters must be defined');
     }
-    const dataRows = SqlDataQuery.fromSql(this.name, this.bucketParameters, sql, options);
+    const dataRows = SqlDataQuery.fromSql(this.name, this.bucketParameters, sql, options, this.compatibility);
 
     this.dataQueries.push(dataRows);
 

packages/sync-rules/src/SqlDataQuery.ts

@@ -12,13 +12,20 @@ import { TablePattern } from './TablePattern.js';
 import { TableQuerySchema } from './TableQuerySchema.js';
 import { EvaluationResult, ParameterMatchClause, QuerySchema, SqliteRow } from './types.js';
 import { getBucketId, isSelectStatement } from './utils.js';
+import { CompatibilityContext } from './quirks.js';
 
 export interface SqlDataQueryOptions extends BaseSqlDataQueryOptions {
   filter: ParameterMatchClause;
 }
 
 export class SqlDataQuery extends BaseSqlDataQuery {
-  static fromSql(descriptorName: string, bucketParameters: string[], sql: string, options: SyncRulesOptions) {
+  static fromSql(
+    descriptorName: string,
+    bucketParameters: string[],
+    sql: string,
+    options: SyncRulesOptions,
+    compatibility: CompatibilityContext
+  ) {
     const parsed = parse(sql, { locationTracking: true });
     const schema = options.schema;
 
@@ -67,6 +74,7 @@ export class SqlDataQuery extends BaseSqlDataQuery {
       table: alias,
       parameterTables: ['bucket'],
       valueTables: [alias],
+      compatibilityContext: compatibility,
       sql,
       schema: querySchema
     });

packages/sync-rules/src/SqlParameterQuery.ts

@@ -29,6 +29,7 @@ import {
   SqliteRow
 } from './types.js';
 import { filterJsonRow, getBucketId, isJsonValue, isSelectStatement, normalizeParameterValue } from './utils.js';
+import { CompatibilityContext } from './quirks.js';
 
 export interface SqlParameterQueryOptions {
   sourceTable: TablePattern;
@@ -122,6 +123,7 @@ export class SqlParameterQuery {
       sql,
       supportsExpandingParameters: true,
       supportsParameterExpressions: true,
+      compatibilityContext: CompatibilityContext.ofFixedQuirks(options.fixedQuirks),
       schema: querySchema
     });
     tools.checkSpecificNameCase(tableRef);

packages/sync-rules/src/SqlSyncRules.ts

@@ -194,7 +194,8 @@ export class SqlSyncRules implements SyncRules {
       const queryOptions: QueryParseOptions = {
         ...options,
         accept_potentially_dangerous_queries,
-        priority: parseOptionPriority
+        priority: parseOptionPriority,
+        fixedQuirks: fixedQuirks
       };
       const parameters = value.get('parameters', true) as unknown;
       const dataQueries = value.get('data', true) as unknown;

packages/sync-rules/src/StaticSqlParameterQuery.ts

@@ -5,6 +5,7 @@ import { SqlTools } from './sql_filters.js';
 import { checkUnsupportedFeatures, isClauseError, isParameterValueClause, sqliteBool } from './sql_support.js';
 import { ParameterValueClause, QueryParseOptions, RequestParameters, SqliteJsonValue } from './types.js';
 import { getBucketId, isJsonValue } from './utils.js';
+import { CompatibilityContext } from './quirks.js';
 
 export interface StaticSqlParameterQueryOptions {
   sql: string;
@@ -39,6 +40,7 @@ export class StaticSqlParameterQuery {
       table: undefined,
       parameterTables: ['token_parameters', 'user_parameters'],
       supportsParameterExpressions: true,
+      compatibilityContext: CompatibilityContext.ofFixedQuirks(options.fixedQuirks),
       sql
     });
     const where = q.where;

packages/sync-rules/src/TableValuedFunctionSqlParameterQuery.ts

@@ -2,7 +2,7 @@ import { FromCall, SelectedColumn, SelectFromStatement } from 'pgsql-ast-parser'
 import { SqlRuleError } from './errors.js';
 import { SqlTools } from './sql_filters.js';
 import { checkUnsupportedFeatures, isClauseError, isParameterValueClause, sqliteBool } from './sql_support.js';
-import { TABLE_VALUED_FUNCTIONS, TableValuedFunction } from './TableValuedFunctions.js';
+import { generateTableValuedFunctions, TableValuedFunction } from './TableValuedFunctions.js';
 import {
   ParameterValueClause,
   ParameterValueSet,
@@ -13,6 +13,7 @@ import {
 } from './types.js';
 import { getBucketId, isJsonValue } from './utils.js';
 import { BucketDescription, BucketPriority, DEFAULT_BUCKET_PRIORITY } from './BucketDescription.js';
+import { CompatibilityContext } from './quirks.js';
 
 export interface TableValuedFunctionSqlParameterQueryOptions {
   sql: string;
@@ -48,11 +49,13 @@ export class TableValuedFunctionSqlParameterQuery {
     options: QueryParseOptions,
     queryId: string
   ): TableValuedFunctionSqlParameterQuery {
+    const compatibility = CompatibilityContext.ofFixedQuirks(options.fixedQuirks);
     let errors: SqlRuleError[] = [];
 
     errors.push(...checkUnsupportedFeatures(sql, q));
 
-    if (!(call.function.name in TABLE_VALUED_FUNCTIONS)) {
+    const tableValuedFunctions = generateTableValuedFunctions(compatibility);
+    if (!(call.function.name in tableValuedFunctions)) {
       throw new SqlRuleError(`Table-valued function ${call.function.name} is not defined.`, sql, call);
     }
 
@@ -63,6 +66,7 @@ export class TableValuedFunctionSqlParameterQuery {
       table: callTable,
       parameterTables: ['token_parameters', 'user_parameters', callTable],
       supportsParameterExpressions: true,
+      compatibilityContext: compatibility,
       sql
     });
     const where = q.where;
@@ -72,7 +76,7 @@ export class TableValuedFunctionSqlParameterQuery {
     const columns = q.columns ?? [];
     const bucketParameters = columns.map((column) => tools.getOutputName(column));
 
-    const functionImpl = TABLE_VALUED_FUNCTIONS[call.function.name]!;
+    const functionImpl = tableValuedFunctions[call.function.name]!;
     let priority = options.priority;
     let parameterExtractors: Record<string, ParameterValueClause> = {};
 

packages/sync-rules/src/TableValuedFunctions.ts

@@ -1,3 +1,4 @@
+import { CompatibilityContext, Quirk } from './quirks.js';
 import { SqliteJsonValue, SqliteRow, SqliteValue } from './types.js';
 import { jsonValueToSqlite } from './utils.js';
 
@@ -8,38 +9,40 @@ export interface TableValuedFunction {
   documentation: string;
 }
 
-export const JSON_EACH: TableValuedFunction = {
-  name: 'json_each',
-  call(args: SqliteValue[]) {
-    if (args.length != 1) {
-      throw new Error(`json_each expects 1 argument, got ${args.length}`);
-    }
-    const valueString = args[0];
-    if (valueString === null) {
-      return [];
-    } else if (typeof valueString !== 'string') {
-      throw new Error(`Expected json_each to be called with a string, got ${valueString}`);
-    }
-    let values: SqliteJsonValue[] = [];
-    try {
-      values = JSON.parse(valueString);
-    } catch (e) {
-      throw new Error('Expected JSON string');
-    }
-    if (!Array.isArray(values)) {
-      throw new Error(`Expected an array, got ${valueString}`);
-    }
+function jsonEachImplementation(fixedJsonBehavior: boolean): TableValuedFunction {
+  return {
+    name: 'json_each',
+    call(args: SqliteValue[]) {
+      if (args.length != 1) {
+        throw new Error(`json_each expects 1 argument, got ${args.length}`);
+      }
+      const valueString = args[0];
+      if (valueString === null) {
+        return [];
+      } else if (typeof valueString !== 'string') {
+        throw new Error(`Expected json_each to be called with a string, got ${valueString}`);
+      }
+      let values: SqliteJsonValue[] = [];
+      try {
+        values = JSON.parse(valueString);
+      } catch (e) {
+        throw new Error('Expected JSON string');
+      }
+      if (!Array.isArray(values)) {
+        throw new Error(`Expected an array, got ${valueString}`);
+      }
 
-    return values.map((v) => {
-      return {
-        value: jsonValueToSqlite(v)
-      };
-    });
-  },
-  detail: 'Each element of a JSON array',
-  documentation: 'Returns each element of a JSON array as a separate row.'
-};
+      return values.map((v) => {
+        return {
+          value: jsonValueToSqlite(fixedJsonBehavior, v)
+        };
+      });
+    },
+    detail: 'Each element of a JSON array',
+    documentation: 'Returns each element of a JSON array as a separate row.'
+  };
+}
 
-export const TABLE_VALUED_FUNCTIONS: Record<string, TableValuedFunction> = {
-  json_each: JSON_EACH
-};
+export function generateTableValuedFunctions(compatibility: CompatibilityContext): Record<string, TableValuedFunction> {
+  return { json_each: jsonEachImplementation(compatibility.isFixed(Quirk.legacyJsonExtract)) };
+}

packages/sync-rules/src/events/SqlEventDescriptor.ts

@@ -23,7 +23,7 @@ export class SqlEventDescriptor {
   }
 
   addSourceQuery(sql: string, options: SyncRulesOptions): QueryParseResult {
-    const source = SqlEventSourceQuery.fromSql(this.name, sql, options);
+    const source = SqlEventSourceQuery.fromSql(this.name, sql, options, this.compatibility);
 
     // Each source query should be for a unique table
     const existingSourceQuery = this.sourceQueries.find((q) => q.table == source.table);

packages/sync-rules/src/events/SqlEventSourceQuery.ts

@@ -10,6 +10,7 @@ import { TablePattern } from '../TablePattern.js';
 import { TableQuerySchema } from '../TableQuerySchema.js';
 import { EvaluationError, QuerySchema, SqliteJsonRow, SqliteRow } from '../types.js';
 import { isSelectStatement } from '../utils.js';
+import { CompatibilityContext } from '../quirks.js';
 
 export type EvaluatedEventSourceRow = {
   data: SqliteJsonRow;
@@ -24,7 +25,7 @@ export type EvaluatedEventRowWithErrors = {
  * Defines how a Replicated Row is mapped to source parameters for events.
  */
 export class SqlEventSourceQuery extends BaseSqlDataQuery {
-  static fromSql(descriptor_name: string, sql: string, options: SyncRulesOptions) {
+  static fromSql(descriptor_name: string, sql: string, options: SyncRulesOptions, compatibility: CompatibilityContext) {
     const parsed = parse(sql, { locationTracking: true });
     const schema = options.schema;
 
@@ -73,7 +74,8 @@ export class SqlEventSourceQuery extends BaseSqlDataQuery {
       parameterTables: [],
       valueTables: [alias],
       sql,
-      schema: querySchema
+      schema: querySchema,
+      compatibilityContext: compatibility
     });
 
     let extractors: RowValueExtractor[] = [];

packages/sync-rules/src/quirks.ts

@@ -26,9 +26,15 @@ export class Quirk {
     CompatibilityLevel.SYNC_STREAMS
   );
 
+  static legacyJsonExtract = new Quirk(
+    'legacy_json_extract',
+    "Old versions of the sync service used to treat `->> 'foo.bar'` as a two-element JSON path. With this quirk fixed, it follows modern SQLite and treats it as a single key. The `$.` prefix would be required to split on `.`.",
+    CompatibilityLevel.SYNC_STREAMS
+  );
+
   static byName: Record<string, Quirk> = (() => {
     const byName: Record<string, Quirk> = {};
-    for (const entry of [this.nonIso8601Timestamps]) {
+    for (const entry of [this.nonIso8601Timestamps, this.legacyJsonExtract]) {
       byName[entry.name] = entry;
     }