{"objects":[{"metadata":{"identifier":{"packageName":"Matillion.Data Loader.CDC","name":"Create or Refresh External Table (Exchange)","revision":2,"type":"DYNAMIC"},"rootJobReference":{"name":"Create or Refresh External Table - 1 - Process","type":"ORCHESTRATION","parameterMetadata":[{"slot":2,"variableName":"target_database","variableType":"SCALAR","displayName":"Target Database","description":"The Snowflake database containing the external table with change events and the target table.","defaultValue":[{"values":{"1":"[Environment Default]"}}],"defaultValueType":"TEXT","required":true},{"slot":3,"variableName":"target_schema","variableType":"SCALAR","displayName":"Target Schema","description":"The Snowflake schema containing the external table with change events and the target table.","defaultValue":[{"values":{"1":"[Environment Default]"}}],"defaultValueType":"TEXT","required":true},{"slot":4,"variableName":"snowflake_stage","variableType":"SCALAR","displayName":"External Snowflake Stage","description":"The name of the existing external stage which contains the files output by the CDC pipeline.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":5,"variableName":"external_table","variableType":"SCALAR","displayName":"External Table","description":"The external table in Snowflake containing the CDC change events.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":6,"variableName":"cloud_storage_url","variableType":"SCALAR","displayName":"Cloud Storage URL","description":"The URL to the location where the CDC change event files are stored. This should have the format:\n\ns3:////\nor\nazure://.blob.core.windows.net///\nor\ngs:////","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true}]},"orchestrationJobs":["Create or Refresh External Table - 1 - Process","Create or Refresh External Table - 1-1 - Create or refresh"],"transformationJobs":[],"imageMetadata":{"componentIcon":"iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAABW0lEQVR42mNgGAVQoKKiwqekpLQHiP8B8X8K8DdlZeUykh0A1JgPNeAvEP+mAIPNUFRUFCfVAQ0gzUDX11ASkkAzroPMUVBQ0BhQBwCj1FJNTU1ERkaGc0AcgIR/AXHRQDjgORA/BuI/QPyTYEjQKg1AHfEfFB2jDhh1wKgDRh0wIA4A1oorgPxdQMw/smpDsh0AxKuBwRZDLgZ64CnIHCBbPXSVDGfQcsUQohwA1OBLYVMMGb8E1X5BS1RkApfJfyY6FKA+WAQ0YDEQ34BGyXEon1g8A9gY0QaZR7ID0KJkCjQoU8hNDxQ5AGjxJGhwgnyVgQ8DfSwD0xe4TLElaLnCfBAOXK6wCuiA3zA+REw+mygHAIM+kNg4BzrWFqYvYJliTNAyhQIQDliuUB+4TOEHjA/Gy+W9SQkFP6BDuoG4Bx8GOkKV6lFADTDqAIf9DiyBK5TURju++AAACJdjPROUgGMAAAAASUVORK5CYII=","smallIcon":"iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAwUlEQVR42mNgoAZQUlLqA+K/BPAtbW1tNqwGKCoq7gdiOwKWvFZWVpYFMpnINgCIPwHxDQxDiDVATU1NBEh/lJKS4hrhBlyUkZHhJNsAXJKbgfggEK/Fgz8bJqqIBi6VD8KWDsSBiSQQlNqA7BIgHYSOVVRUDD2XqPAFLZWvwulMoCE7gIoPQV0EwwUguYDlijWBSxX6ApcpHAhcrtARvExRD8MAkB+BLnBDxkBDdWDyBF1ACCTMV+AIWKHoxkBNAAC1wGPctYHkbgAAAABJRU5ErkJggg=="},"description":"Please see the full shared job documentation at the link - https://documentation.matillion.com/docs/sharedjobs-cdc","helpHtml":"

Create or Refresh External Table

Please see the full shared job documentation at the link - https://documentation.matillion.com/docs/sharedjobs-cdc

Properties

PropertyTypeDescription
Target DatabaseVariableThe Snowflake database containing the external table with change events and the target table.
Target SchemaVariableThe Snowflake schema containing the external table with change events and the target table.
External Snowflake StageVariableThe name of the existing external stage which contains the files output by the CDC pipeline.
External TableVariableThe external table in Snowflake containing the CDC change events.
Cloud Storage URLVariableThe URL to the location where the CDC change event files are stored. This should have the format:\n\ns3://<bucket>/<prefix>/\nor\nazure://<account>.blob.core.windows.net/<container>/<prefix>/\nor\ngs://<bucket>/<prefix>/
","created":1669030202454},"orchestrationJobs":{"Create or Refresh External Table - 1-1 - Create or refresh":{"id":-2,"revision":17,"created":1669030171354,"timestamp":1669030171354,"components":{"41584":{"id":41584,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1032749985,"x":235,"y":154,"width":32,"height":32,"inputConnectorIDs":[41607],"outputSuccessConnectorIDs":[41604],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Check external table exists"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"SQL Query","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${table_exists_sql}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Scalar Variable Mapping","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"table_type"},"2":{"slot":2,"type":"STRING","value":"TABLE_TYPE"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"[Environment Default]"}}}},"visible":false,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Table Columns","elements":{},"visible":false,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"[Environment Default]"}}}},"visible":false,"status":{},"columnNames":[]},"10":{"slot":10,"name":"Limit","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"100"}}}},"visible":false,"status":{},"columnNames":[]},"11":{"slot":11,"name":"Order By","elements":{},"visible":false,"status":{},"columnNames":[]},"12":{"slot":12,"name":"Sort","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Ascending"}}}},"visible":false,"status":{},"columnNames":[]},"13":{"slot":13,"name":"Filter Conditions","elements":{},"visible":false,"status":{},"columnNames":[]},"14":{"slot":14,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"AND"}}}},"visible":false,"status":{},"columnNames":[]},"20":{"slot":20,"name":"Basic / Advanced","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Advanced"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41585":{"id":41585,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":395,"y":154,"width":32,"height":32,"inputConnectorIDs":[41604],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[46600],"outputFalseConnectorIDs":[41612],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Table exists and is an external table?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"table_type"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"EXTERNAL TABLE"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41586":{"id":41586,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":1440,"y":-80,"width":32,"height":32,"inputConnectorIDs":[41608],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Success - External table refreshed"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41587":{"id":41587,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":1840,"y":240,"width":32,"height":32,"inputConnectorIDs":[41609],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Success - External table created"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41588":{"id":41588,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":75,"y":154,"width":32,"height":32,"inputConnectorIDs":[41615],"outputSuccessConnectorIDs":[41607],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Build table_exists_sql"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"table_exists_sql = \\\nf\"\"\"SELECT \n COALESCE(ANY_VALUE(UPPER(\"TABLE_TYPE\")), 'NO TABLE') \"TABLE_TYPE\"\nFROM \n \"${target_database}\".\"INFORMATION_SCHEMA\".\"TABLES\"\nWHERE \n \"TABLE_SCHEMA\" = '${target_schema}'\n AND \"TABLE_NAME\" = '${external_table}'\"\"\"\n\nprint(f'table_exists_sql: \\n{table_exists_sql}')\ncontext.updateVariable('table_exists_sql', table_exists_sql)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41589":{"id":41589,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":539,"y":234,"width":32,"height":32,"inputConnectorIDs":[41612],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[41611],"outputFalseConnectorIDs":[41613],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"External table name is available?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"table_type"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"NO TABLE"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41590":{"id":41590,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":715,"y":314,"width":32,"height":32,"inputConnectorIDs":[41613],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[41614],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Table exists but is not an external table"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"err_msg = f'A table with the name {external_table!r} exists but has type {table_type!r}. Please try a different name for the external table.'\nraise Exception(err_msg)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41591":{"id":41591,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":715,"y":474,"width":32,"height":32,"inputConnectorIDs":[41614],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Failure - Incorrect name for external table"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41592":{"id":41592,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":1120,"y":240,"width":32,"height":32,"inputConnectorIDs":[41603],"outputSuccessConnectorIDs":[41602],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Build get_stage_url_sql"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"get_stage_url_sql = \\\nf\"\"\"SELECT \n \"STAGE_URL\"\nFROM \"{target_database}\".\"INFORMATION_SCHEMA\".\"STAGES\"\nWHERE\n \"STAGE_TYPE\" = 'External Named'\n AND \"STAGE_CATALOG\" = '{target_database}'\n AND \"STAGE_SCHEMA\" = '{target_schema}'\n AND \"STAGE_NAME\" = '{snowflake_stage}'\"\"\"\n\nprint(f'get_stage_url_sql: \\n{get_stage_url_sql}')\ncontext.updateVariable('get_stage_url_sql', get_stage_url_sql)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41593":{"id":41593,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1032749985,"x":1280,"y":240,"width":32,"height":32,"inputConnectorIDs":[41602],"outputSuccessConnectorIDs":[41606],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Get url for external stage"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"SQL Query","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${get_stage_url_sql}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Scalar Variable Mapping","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"stage_url"},"2":{"slot":2,"type":"STRING","value":"STAGE_URL"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"[Environment Default]"}}}},"visible":false,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Table Columns","elements":{},"visible":false,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"[Environment Default]"}}}},"visible":false,"status":{},"columnNames":[]},"10":{"slot":10,"name":"Limit","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"100"}}}},"visible":false,"status":{},"columnNames":[]},"11":{"slot":11,"name":"Order By","elements":{},"visible":false,"status":{},"columnNames":[]},"12":{"slot":12,"name":"Sort","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Ascending"}}}},"visible":false,"status":{},"columnNames":[]},"13":{"slot":13,"name":"Filter Conditions","elements":{},"visible":false,"status":{},"columnNames":[]},"14":{"slot":14,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"AND"}}}},"visible":false,"status":{},"columnNames":[]},"20":{"slot":20,"name":"Basic / Advanced","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Advanced"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41594":{"id":41594,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":1440,"y":240,"width":32,"height":32,"inputConnectorIDs":[41606],"outputSuccessConnectorIDs":[41605],"outputFailureConnectorIDs":[41617],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Check for external table relative_path"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"# standardise GCS protocol to 'gcs' to match Snowflake\nif cloud_storage_url.startswith('gs://'):\n print(\"Updating protocol from 'gs' to 'gcs' to match Snowflake's format\")\n cloud_storage_url = f'gcs://{cloud_storage_url[5:]}'\n\nprint(f'Cloud Storage URL: {cloud_storage_url!r}')\nprint(f'Snowflake external stage: {stage_url!r}')\nprint()\n\nif not stage_url:\n err_msg = f'Unable to lookup the URL for the Snowflake external stage \"{target_database}\".\"{target_schema}\".\"{snowflake_stage}\"'\n raise Exception(err_msg)\n\nif not cloud_storage_url.startswith(stage_url):\n err_msg = 'Unable to proceed as the Cloud Storage Location is not within the Snowflake external stage.'\n raise Exception(err_msg)\n\nrelative_path = cloud_storage_url[len(stage_url):]\n\nif relative_path:\n print(f'relative_path: {relative_path!r}')\n context.updateVariable('relative_path', relative_path)\nelse:\n print(f'No relative_path as stage URL is the same as the Cloud Storage URL')\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41595":{"id":41595,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":960,"y":240,"width":32,"height":32,"inputConnectorIDs":[41611],"outputSuccessConnectorIDs":[41603],"outputFailureConnectorIDs":[41616],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Validate cloud_storage_url"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"def validate_host_path(host_path: str, num_parts: int) -> bool:\n \"\"\"\n Validate that the host_path can be split into num_parts, \n and that no parts are empty strings\n \"\"\"\n host_path_list = host_path.split('/', num_parts - 1)\n \n if len(host_path_list) != num_parts:\n return False\n \n if '' in host_path_list:\n return False\n \n return True\n\n \nif not cloud_storage_url[-1:] == '/':\n cloud_storage_url = cloud_storage_url + '/'\n context.updateVariable('cloud_storage_url', cloud_storage_url)\n \nprint(f'cloud_storage_url: {cloud_storage_url!r}')\nprint()\n\n# split the cloud_storage_url into protocol and host/path\nurl_list = cloud_storage_url.split('://', 1)\n\n# check we could split the string and extract a protocol\nif len(url_list) == 1:\n err_msg = f'Unable to determine the protocol from the cloud_storage_url {cloud_storage_url!r}'\n raise Exception(err_msg)\n\n# unpack the list\nprotocol, host_path = url_list\nprotocol = protocol.lower()\n\n# check protocol is supported\nsupported_protocols = ('azure', 's3', 'gs', 'gcs')\nif protocol not in supported_protocols:\n err_msg = f'The protocol {protocol!r} was identifed from the cloud_storage_url {cloud_storage_url!r}, but the only supported protocols are {\", \".join(supported_protocols)}'\n raise Exception(err_msg)\n\n# s3 cloud_storage_urls must have the format s3://bucket/prefix/\nif protocol == 's3' and not validate_host_path(host_path, 2):\n err_msg = f'A cloud_storage_url for S3 must have the format \"s3://bucket/prefix/\" but is actually {cloud_storage_url!r}'\n raise Exception(err_msg)\n \n# azure cloud_storage_urls must have the format azure://account/container/prefix/\nelif protocol == 'azure' and not validate_host_path(host_path, 3):\n err_msg = f'A cloud_storage_url for Azure must have the format \"azure://account/container/prefix/\" but is actually {cloud_storage_url!r}'\n raise Exception(err_msg)\n\n# google cloud_storage_urls must have the format gs://bucket/prefix/ or gcs://bucket/prefix/\nelif protocol in ('gs', 'gcs') and not validate_host_path(host_path, 2):\n err_msg = f'A cloud_storage_url for Google Cloud Storage must have the format \"gs://bucket/prefix/\" or \"gcs://bucket/prefix/\" but is actually {cloud_storage_url!r}'\n raise Exception(err_msg)\n "}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41596":{"id":41596,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":960,"y":400,"width":32,"height":32,"inputConnectorIDs":[41616],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Failure - Invalid cloud_storage_url"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41597":{"id":41597,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":1440,"y":400,"width":32,"height":32,"inputConnectorIDs":[41617],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Failure - Cloud storage URL not within external stage"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41598":{"id":41598,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":-80,"y":160,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[41615],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41599":{"id":41599,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1211855512,"x":1280,"y":-80,"width":32,"height":32,"inputConnectorIDs":[46649],"outputSuccessConnectorIDs":[41608],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Refresh External Table"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${external_table}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Relative Subpath","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41601":{"id":41601,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":1364466516,"x":1680,"y":240,"width":32,"height":32,"inputConnectorIDs":[41605],"outputSuccessConnectorIDs":[41609],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Create External Table"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Create/Replace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Create"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"New Table Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${external_table}"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Partition Columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"partition_columns"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"name"},"2":{"slot":2,"type":"GRID","value":"type"},"3":{"slot":3,"type":"GRID","value":"size"},"4":{"slot":4,"type":"GRID","value":"precision"},"5":{"slot":5,"type":"GRID","value":"partition_expression"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Pattern","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${pattern}"}}}},"visible":true,"status":{},"columnNames":[]},"10":{"slot":10,"name":"Format","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"[Custom]"}}}},"visible":true,"status":{},"columnNames":[]},"11":{"slot":11,"name":"File Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${file_type}"}}}},"visible":true,"status":{},"columnNames":[]},"12":{"slot":12,"name":"Compression","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"AUTO"}}}},"visible":true,"status":{},"columnNames":[]},"20":{"slot":20,"name":"Record Delimiter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"21":{"slot":21,"name":"Field Delimiter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"22":{"slot":22,"name":"Skip Header","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":""}}}},"visible":true,"status":{},"columnNames":[]},"26":{"slot":26,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"27":{"slot":27,"name":"Escape","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"CHAR_OR_ESCAPE_SEQUENCE","value":""}}}},"visible":true,"status":{},"columnNames":[]},"28":{"slot":28,"name":"Escape Unenclosed Field","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"CHAR_OR_ESCAPE_SEQUENCE","value":""}}}},"visible":true,"status":{},"columnNames":[]},"29":{"slot":29,"name":"Trim Space","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"False"}}}},"visible":true,"status":{},"columnNames":[]},"30":{"slot":30,"name":"Field Optionally Enclosed","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"CHAR","value":""}}}},"visible":true,"status":{},"columnNames":[]},"31":{"slot":31,"name":"Null If","elements":{},"visible":true,"status":{},"columnNames":[]},"32":{"slot":32,"name":"Error On Column Count Mismatch","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"False"}}}},"visible":true,"status":{},"columnNames":[]},"33":{"slot":33,"name":"Empty Field as Null","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"True"}}}},"visible":true,"status":{},"columnNames":[]},"34":{"slot":34,"name":"Encoding Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"35":{"slot":35,"name":"Skip Blank Lines","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"False"}}}},"visible":true,"status":{},"columnNames":[]},"36":{"slot":36,"name":"Replace Invalid Characters","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"False"}}}},"visible":true,"status":{},"columnNames":[]},"40":{"slot":40,"name":"Enable Octal","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"False"}}}},"visible":false,"status":{},"columnNames":[]},"41":{"slot":41,"name":"Allow Duplicates","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"False"}}}},"visible":false,"status":{},"columnNames":[]},"42":{"slot":42,"name":"Strip Outer Array","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"False"}}}},"visible":false,"status":{},"columnNames":[]},"43":{"slot":43,"name":"Strip Null Values","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"False"}}}},"visible":false,"status":{},"columnNames":[]},"44":{"slot":44,"name":"Ignore UTF8 Errors","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"False"}}}},"visible":false,"status":{},"columnNames":[]},"45":{"slot":45,"name":"Nest Columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"False"}}}},"visible":false,"status":{},"columnNames":[]},"46":{"slot":46,"name":"Null If","elements":{},"visible":false,"status":{},"columnNames":[]},"47":{"slot":47,"name":"Trim Space","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"False"}}}},"visible":false,"status":{},"columnNames":[]},"50":{"slot":50,"name":"Null If","elements":{},"visible":false,"status":{},"columnNames":[]},"51":{"slot":51,"name":"Trim Space","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"False"}}}},"visible":false,"status":{},"columnNames":[]},"60":{"slot":60,"name":"Ignore UTF8 Errors","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"False"}}}},"visible":false,"status":{},"columnNames":[]},"61":{"slot":61,"name":"Preserve Space","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"False"}}}},"visible":false,"status":{},"columnNames":[]},"62":{"slot":62,"name":"Strip Outer Element","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"False"}}}},"visible":false,"status":{},"columnNames":[]},"63":{"slot":63,"name":"Disable Snowflake Data","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"False"}}}},"visible":false,"status":{},"columnNames":[]},"64":{"slot":64,"name":"Disable Auto Convert","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"False"}}}},"visible":false,"status":{},"columnNames":[]},"66":{"slot":66,"name":"Null If","elements":{},"visible":false,"status":{},"columnNames":[]},"67":{"slot":67,"name":"Trim Space","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"False"}}}},"visible":false,"status":{},"columnNames":[]},"70":{"slot":70,"name":"Null If","elements":{},"visible":false,"status":{},"columnNames":[]},"71":{"slot":71,"name":"Trim Space","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"False"}}}},"visible":false,"status":{},"columnNames":[]},"100":{"slot":100,"name":"Stage Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"101":{"slot":101,"name":"Stage Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"102":{"slot":102,"name":"Stage","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${snowflake_stage}"}}}},"visible":true,"status":{},"columnNames":[]},"103":{"slot":103,"name":"Relative Path","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${relative_path}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"46599":{"id":46599,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":356330947,"x":960,"y":-80,"width":32,"height":32,"inputConnectorIDs":[46600],"outputSuccessConnectorIDs":[46611],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Read external table metadata"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Grid Variable","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"table_metadata"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Grid Variable Mapping","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Name"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"Type"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"Size"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"Precision"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${external_table}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"46610":{"id":46610,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":1120,"y":-80,"width":32,"height":32,"inputConnectorIDs":[46611],"outputSuccessConnectorIDs":[46649],"outputFailureConnectorIDs":[46671],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Check external table metadata"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"from pprint import pprint\n\n\ndef convert_grid_to_set(grid: list) -> set:\n \"\"\"Convert grid variable to a set.\"\"\"\n new_set = set()\n for row in grid:\n new_set.add(tuple(row))\n return new_set\n\n\n# build expected metadata from partition_columns\npartition_columns = context.getGridVariable('partition_columns') or []\nexpected_metadata_set = {\n (name, datatype, size, precision) for name, datatype, size, precision, *_ in partition_columns\n}\nexpected_metadata_set.add(\n ('VALUE', 'VARIANT', '0', '0')\n)\n\nprint('expected_metadata_set:')\npprint(expected_metadata_set, width=150)\nprint()\n\n# build actual metadata from table_metadata\ntable_metadata = context.getGridVariable('table_metadata') or []\nactual_metadata_set = convert_grid_to_set(table_metadata)\n\nprint('actual_metadata_set:')\npprint(actual_metadata_set, width=150)\nprint()\n\n# check for missing or unexpected columns\nactual_not_in_expected = actual_metadata_set - expected_metadata_set\nexpected_not_in_actual = expected_metadata_set - actual_metadata_set\n\n# if all metadata matches, exit\nif not(actual_not_in_expected or expected_not_in_actual):\n print('Expected metadata matches actual metadata for external table.')\n exit()\n\nprint('Unexpected columns in external table:')\npprint(actual_not_in_expected)\nprint()\n\nprint('Missing columns from external table:')\npprint(expected_not_in_actual)\nprint()\n\nerr_msg = (f'The metadata for \"{target_database}\".\"{target_schema}\".\"{external_table}\" '\n 'does not match the expected metadata. \\nIf this table was created by '\n 'revision 1 of this shared job, all external and target tables must be '\n 'dropped before using the revision 2 shared jobs.')\nraise Exception(err_msg)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"46663":{"id":46663,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":1120,"y":-320,"width":32,"height":32,"inputConnectorIDs":[46671],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Failure - Invalid metadata for external table"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"41602":{"id":41602,"sourceID":41592,"targetID":41593},"41603":{"id":41603,"sourceID":41595,"targetID":41592},"41604":{"id":41604,"sourceID":41584,"targetID":41585},"41605":{"id":41605,"sourceID":41594,"targetID":41601},"41606":{"id":41606,"sourceID":41593,"targetID":41594},"41607":{"id":41607,"sourceID":41588,"targetID":41584},"41608":{"id":41608,"sourceID":41599,"targetID":41586},"41609":{"id":41609,"sourceID":41601,"targetID":41587},"46611":{"id":46611,"sourceID":46599,"targetID":46610},"46649":{"id":46649,"sourceID":46610,"targetID":41599}},"failureConnectors":{"41616":{"id":41616,"sourceID":41595,"targetID":41596},"41617":{"id":41617,"sourceID":41594,"targetID":41597},"46671":{"id":46671,"sourceID":46610,"targetID":46663}},"unconditionalConnectors":{"41614":{"id":41614,"sourceID":41590,"targetID":41591},"41615":{"id":41615,"sourceID":41598,"targetID":41588}},"trueConnectors":{"41611":{"id":41611,"sourceID":41589,"targetID":41595},"46600":{"id":46600,"sourceID":41585,"targetID":46599}},"falseConnectors":{"41612":{"id":41612,"sourceID":41585,"targetID":41589},"41613":{"id":41613,"sourceID":41589,"targetID":41590}},"iterationConnectors":{},"noteConnectors":{},"canUndo":true,"undoCommand":"Delete","undoCreated":1669029934664,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"41581":{"id":41581,"x":6,"y":6,"width":788,"height":388,"text":"**Create or Refresh external table**\n\nIf required, create the external table over the Snowflake stage containing the files output by the CDC agent. The table will be created with partition columns to enable efficient querying for new data.\n\nIf the external table already exists, it will be refreshed to contain any new files.","colour":"e6e63c"},"41582":{"id":41582,"x":884,"y":85,"width":629,"height":232,"text":"**Check for relative path**\n\nValidate the __cloud_storage_url__\n\nQuery information_schema to get external stage_url\n\nCompare to the __cloud_storage_url__to the __stage_url__determine whether the external table needs a __relative_path__.","colour":"e6e63c"},"41583":{"id":41583,"x":1603,"y":84,"width":149,"height":231,"text":"**Create external table**","colour":"e6e63c"},"46653":{"id":46653,"x":884,"y":-234,"width":470,"height":228,"text":"**Refresh existing external table**\n\nCheck the external table has the expected metadata for v2.0.0 of this job.\nIf so, refresh to pickup new files.","colour":"e6e63c"}},"variables":{"pattern":{"definition":{"name":"pattern","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"file_type":{"definition":{"name":"file_type","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"stage_url":{"definition":{"name":"stage_url","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"table_type":{"definition":{"name":"table_type","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":null},"relative_path":{"definition":{"name":"relative_path","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"target_schema":{"definition":{"name":"target_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":null},"external_table":{"definition":{"name":"external_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":null},"snowflake_stage":{"definition":{"name":"snowflake_stage","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":null},"target_database":{"definition":{"name":"target_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":null},"table_exists_sql":{"definition":{"name":"table_exists_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":null},"cloud_storage_url":{"definition":{"name":"cloud_storage_url","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"get_stage_url_sql":{"definition":{"name":"get_stage_url_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""}},"grids":{"table_metadata":{"definition":{"name":"table_metadata","scope":"TASKBATCH","definitions":[{"name":"name","type":"TEXT"},{"name":"type","type":"TEXT"},{"name":"size","type":"TEXT"},{"name":"precision","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[]},"partition_columns":{"definition":{"name":"partition_columns","scope":"TASKBATCH","definitions":[{"name":"name","type":"TEXT"},{"name":"type","type":"TEXT"},{"name":"size","type":"TEXT"},{"name":"precision","type":"TEXT"},{"name":"partition_expression","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]}}},"Create or Refresh External Table - 1 - Process":{"id":-2,"revision":6,"created":1669030182149,"timestamp":1669030182149,"components":{"41569":{"id":41569,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":-1360,"y":0,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[50145],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Start","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41570":{"id":41570,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":-960,"y":0,"width":32,"height":32,"inputConnectorIDs":[50183],"outputSuccessConnectorIDs":[41576],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Update [Environment Default] values"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"if target_database == '[Environment Default]':\n print(f'Updating target_database from [Environment Default] to: {environment_database}')\n context.updateVariable('target_database', environment_database)\n\nif target_schema == '[Environment Default]':\n print(f'Updating target_schema from [Environment Default] to: {environment_default_schema}')\n context.updateVariable('target_schema', environment_default_schema)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41571":{"id":41571,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":-720,"y":0,"width":32,"height":32,"inputConnectorIDs":[41576],"outputSuccessConnectorIDs":[41575],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Build table names"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"external_table = external_table.upper()\nexternal_metadata_table = f'{external_table}_VERSION_METADATA'\n\nprint(f'external_table: {external_table}')\ncontext.updateVariable('external_table', external_table)\n\nprint(f'external_metadata_table: {external_metadata_table}')\ncontext.updateVariable('external_metadata_table', external_metadata_table)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41572":{"id":41572,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":-480,"y":0,"width":32,"height":32,"inputConnectorIDs":[41575],"outputSuccessConnectorIDs":[41577],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Create or refresh external table"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Create or Refresh External Table - 1-1 - Create or refresh"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"cloud_storage_url"},"2":{"slot":2,"type":"STRING","value":"${cloud_storage_url}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"target_database"},"2":{"slot":2,"type":"STRING","value":"${target_database}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"target_schema"},"2":{"slot":2,"type":"STRING","value":"${target_schema}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"snowflake_stage"},"2":{"slot":2,"type":"STRING","value":"${snowflake_stage}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"external_table"},"2":{"slot":2,"type":"STRING","value":"${external_table}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"file_type"},"2":{"slot":2,"type":"STRING","value":"AVRO"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DATABASE"},"2":{"slot":2,"type":"STRING","value":"VARCHAR"},"3":{"slot":3,"type":"STRING","value":"16777216"},"4":{"slot":4,"type":"STRING","value":"0"},"5":{"slot":5,"type":"STRING","value":"SPLIT_PART(SPLIT_PART(\"METADATA$FILENAME\", '/', -9), '=', -1)"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"SCHEMA"},"2":{"slot":2,"type":"STRING","value":"VARCHAR"},"3":{"slot":3,"type":"STRING","value":"16777216"},"4":{"slot":4,"type":"STRING","value":"0"},"5":{"slot":5,"type":"STRING","value":"SPLIT_PART(SPLIT_PART(\"METADATA$FILENAME\", '/', -8), '=', -1)"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"TABLE"},"2":{"slot":2,"type":"STRING","value":"VARCHAR"},"3":{"slot":3,"type":"STRING","value":"16777216"},"4":{"slot":4,"type":"STRING","value":"0"},"5":{"slot":5,"type":"STRING","value":"SPLIT_PART(SPLIT_PART(\"METADATA$FILENAME\", '/', -7), '=', -1)"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"VERSION"},"2":{"slot":2,"type":"STRING","value":"NUMBER"},"3":{"slot":3,"type":"STRING","value":"38"},"4":{"slot":4,"type":"STRING","value":"0"},"5":{"slot":5,"type":"STRING","value":"CAST(SPLIT_PART(SPLIT_PART(\"METADATA$FILENAME\", '/', -6), '=', -1) AS NUMBER(38, 0))"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"DATE_HOUR"},"2":{"slot":2,"type":"STRING","value":"TIMESTAMP"},"3":{"slot":3,"type":"STRING","value":"9"},"4":{"slot":4,"type":"STRING","value":"0"},"5":{"slot":5,"type":"STRING","value":"TO_TIMESTAMP(SPLIT_PART(SPLIT_PART(\"METADATA$FILENAME\", '/', -5), '=', -1) || SPLIT_PART(SPLIT_PART(\"METADATA$FILENAME\", '/', -4), '=', -1) || SPLIT_PART(SPLIT_PART(\"METADATA$FILENAME\", '/', -3), '=', -1) || SPLIT_PART(SPLIT_PART(\"METADATA$FILENAME\", '/', -2), '=', -1), 'YYYYMMddHH')"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"FILENAME"},"2":{"slot":2,"type":"STRING","value":"VARCHAR"},"3":{"slot":3,"type":"STRING","value":"16777216"},"4":{"slot":4,"type":"STRING","value":"0"},"5":{"slot":5,"type":"STRING","value":"SPLIT_PART(\"METADATA$FILENAME\", '/', -1)"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"FILEPATH"},"2":{"slot":2,"type":"STRING","value":"VARCHAR"},"3":{"slot":3,"type":"STRING","value":"16777216"},"4":{"slot":4,"type":"STRING","value":"0"},"5":{"slot":5,"type":"STRING","value":"\"METADATA$FILENAME\""}}}},"values":{"1":{"slot":1,"type":"STRING","value":"partition_columns"},"2":{"slot":2,"type":"STRING","value":"values"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Create or Refresh External Table (Exchange)"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Data Loader.CDC"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"2"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41573":{"id":41573,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":-320,"y":0,"width":32,"height":32,"inputConnectorIDs":[41577],"outputSuccessConnectorIDs":[41578],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Create or refresh external metadata table"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Create or Refresh External Table - 1-1 - Create or refresh"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"cloud_storage_url"},"2":{"slot":2,"type":"STRING","value":"${cloud_storage_url}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"target_database"},"2":{"slot":2,"type":"STRING","value":"${target_database}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"target_schema"},"2":{"slot":2,"type":"STRING","value":"${target_schema}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"snowflake_stage"},"2":{"slot":2,"type":"STRING","value":"${snowflake_stage}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"external_table"},"2":{"slot":2,"type":"STRING","value":"${external_metadata_table}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"file_type"},"2":{"slot":2,"type":"STRING","value":"JSON"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"pattern"},"2":{"slot":2,"type":"STRING","value":".*/schema\\.avsc"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DATABASE"},"2":{"slot":2,"type":"STRING","value":"VARCHAR"},"3":{"slot":3,"type":"STRING","value":"16777216"},"4":{"slot":4,"type":"STRING","value":"0"},"5":{"slot":5,"type":"STRING","value":"SPLIT_PART(SPLIT_PART(\"METADATA$FILENAME\", '/', -5), '=', -1)"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"SCHEMA"},"2":{"slot":2,"type":"STRING","value":"VARCHAR"},"3":{"slot":3,"type":"STRING","value":"16777216"},"4":{"slot":4,"type":"STRING","value":"0"},"5":{"slot":5,"type":"STRING","value":"SPLIT_PART(SPLIT_PART(\"METADATA$FILENAME\", '/', -4), '=', -1)"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"TABLE"},"2":{"slot":2,"type":"STRING","value":"VARCHAR"},"3":{"slot":3,"type":"STRING","value":"16777216"},"4":{"slot":4,"type":"STRING","value":"0"},"5":{"slot":5,"type":"STRING","value":"SPLIT_PART(SPLIT_PART(\"METADATA$FILENAME\", '/', -3), '=', -1)"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"VERSION"},"2":{"slot":2,"type":"STRING","value":"NUMBER"},"3":{"slot":3,"type":"STRING","value":"38"},"4":{"slot":4,"type":"STRING","value":"0"},"5":{"slot":5,"type":"STRING","value":"CAST(SPLIT_PART(SPLIT_PART(\"METADATA$FILENAME\", '/', -2), '=', -1) AS NUMBER(38, 0))"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"FILENAME"},"2":{"slot":2,"type":"STRING","value":"VARCHAR"},"3":{"slot":3,"type":"STRING","value":"16777216"},"4":{"slot":4,"type":"STRING","value":"0"},"5":{"slot":5,"type":"STRING","value":"SPLIT_PART(\"METADATA$FILENAME\", '/', -1)"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"FILEPATH"},"2":{"slot":2,"type":"STRING","value":"VARCHAR"},"3":{"slot":3,"type":"STRING","value":"16777216"},"4":{"slot":4,"type":"STRING","value":"0"},"5":{"slot":5,"type":"STRING","value":"\"METADATA$FILENAME\""}}}},"values":{"1":{"slot":1,"type":"STRING","value":"partition_columns"},"2":{"slot":2,"type":"STRING","value":"values"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Create or Refresh External Table (Exchange)"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Data Loader.CDC"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"2"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41574":{"id":41574,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":-160,"y":0,"width":32,"height":32,"inputConnectorIDs":[41578],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Success"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"50138":{"id":50138,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":-1200,"y":0,"width":32,"height":32,"inputConnectorIDs":[50145],"outputSuccessConnectorIDs":[50183],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Print version"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"print('Version: 2.0.1')"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"41575":{"id":41575,"sourceID":41571,"targetID":41572},"41576":{"id":41576,"sourceID":41570,"targetID":41571},"41577":{"id":41577,"sourceID":41572,"targetID":41573},"41578":{"id":41578,"sourceID":41573,"targetID":41574},"50183":{"id":50183,"sourceID":50138,"targetID":41570}},"failureConnectors":{},"unconditionalConnectors":{"50145":{"id":50145,"sourceID":41569,"targetID":50138}},"trueConnectors":{},"falseConnectors":{},"iterationConnectors":{},"noteConnectors":{},"canUndo":true,"undoCommand":"Set Parameter","undoCreated":1669029665649,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"41565":{"id":41565,"x":-1034,"y":-154,"width":150,"height":228,"text":"**Update [Environment Default] values**\n\nReplace any [Environment Default] parameters with the actual value","colour":"e6e63c"},"41566":{"id":41566,"x":-796,"y":-153,"width":151,"height":228,"text":"**Build table names**\n\nEnsure external table name is upper case.","colour":"e6e63c"},"41567":{"id":41567,"x":-2074,"y":-155,"width":627,"height":387,"text":"**Change log**\n\n1.0.0 -----------------------------------------------------------------------------------------------------------\n- General availability\n- Support for S3 and Azure Blob Storage locations\n- Support for schema drift\n\n1.1.0 -----------------------------------------------------------------------------------------------------------\n- Support for Google Cloud Storage locations\n\n2.0.0 -----------------------------------------------------------------------------------------------------------\n- Add separate FILENAME and FILEPATH columns to the external tables\n\n2.0.1 -----------------------------------------------------------------------------------------------------------\n- Apply pattern to metadata table to only include 'schema.avsc' files\n","colour":"00ce4f"},"41568":{"id":41568,"x":-553,"y":-154,"width":307,"height":229,"text":"**Create or refresh external tables**\n\nCreate or refresh the following tables\n\n- \n- _VERSION_METADATA","colour":"e6e63c"},"50196":{"id":50196,"x":-1275,"y":-154,"width":152,"height":229,"text":"**Print version**\n\nPrint shared job version to the task history, to help support cases","colour":"e6e63c"}},"variables":{"target_schema":{"definition":{"name":"target_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"external_table":{"definition":{"name":"external_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":null},"snowflake_stage":{"definition":{"name":"snowflake_stage","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":null},"target_database":{"definition":{"name":"target_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"cloud_storage_url":{"definition":{"name":"cloud_storage_url","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"external_metadata_table":{"definition":{"name":"external_metadata_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""}},"grids":{}}},"transformationJobs":{}},{"metadata":{"identifier":{"packageName":"Matillion.Data Loader.CDC","name":"Sync All Tables (Exchange)","revision":3,"type":"DYNAMIC"},"rootJobReference":{"name":"Sync All Tables - 1 - Iterate over tables","type":"ORCHESTRATION","parameterMetadata":[{"slot":2,"variableName":"cloud_storage_url","variableType":"SCALAR","displayName":"Cloud Storage URL","description":"The URL of the location where the CDC change event files are stored. This should have the format:\n\ns3:////\nor\nazure://.blob.core.windows.net///\nor\ngs:////","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":3,"variableName":"warehouse","variableType":"SCALAR","displayName":"Warehouse","description":"The Snowflake warehouse used to process the data.","defaultValue":[{"values":{"1":"[Environment Default]"}}],"defaultValueType":"TEXT","required":true},{"slot":4,"variableName":"target_database","variableType":"SCALAR","displayName":"Target Database","description":"The Snowflake database containing the external table with change events and the target table.","defaultValue":[{"values":{"1":"[Environment Default]"}}],"defaultValueType":"TEXT","required":true},{"slot":5,"variableName":"target_schema","variableType":"SCALAR","displayName":"Target Schema","description":"The Snowflake schema containing the external table with change events and the target table.","defaultValue":[{"values":{"1":"[Environment Default]"}}],"defaultValueType":"TEXT","required":true},{"slot":6,"variableName":"snowflake_stage","variableType":"SCALAR","displayName":"Snowflake External Stage","description":"The name of the existing external stage which contains the files output by the CDC pipeline.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":7,"variableName":"external_table","variableType":"SCALAR","displayName":"External Table","description":"The external table in Snowflake containing the CDC change events.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":8,"variableName":"concurrency","variableType":"SCALAR","displayName":"Concurrency","description":"How to iterate over the source tables for which there are CDC event data:\n\nSequential: Iterations are done in sequence, waiting for each to complete before starting the next. This is the default.\nConcurrent: Iterations are run concurrently.","defaultValue":[{"values":{"1":"Sequential"}}],"defaultValueType":"TEXT","required":true},{"slot":9,"variableName":"target_prefix","variableType":"SCALAR","displayName":"Target Table Prefix","description":"A prefix to prepend to the source table name to generate the target table name. If no Target Table Prefix is specified, the target table will have the same name as the source table.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":false},{"slot":10,"variableName":"fully_qualify_target_table","variableType":"SCALAR","displayName":"Fully Qualify Target Table (Recommended)","description":"Optionally includes the source database and schema in the target table name.\n\nY: The target table name will have the format \n__\n\nN: The target table name will have the format \n\n\nIt is strongly recommended to leave this set to 'Y', unless you are confident that your source table names will always be unique.\n\nIf the target table name is not fully qualified, and there are source tables with the same name in different databases or schemas, this job will see them as the same table and attempt to merge their metadata and contents.","defaultValue":[{"values":{"1":"Y"}}],"defaultValueType":"TEXT","required":true},{"slot":11,"variableName":"transformation_type","variableType":"SCALAR","displayName":"Transformation Type","description":"The type of transformation used when applying the change events to the target table.\n\nCopy Table: The target table will be maintained as a copy of the source table\nCopy Table with Soft Deletes: Same as Copy Table, but records deleted in the source table will be retained in the target table.\nChange Log: All change events will be extracted and appended to the target table\n\nNote: A primary key is required on the source table for Copy Table and Copy Table with Soft Deletes transformations. The primary key is used by the shared job to merge updates into the target table. If the source table does not have a primary key, the transformation type will be updated to Change Log.\n\nWarning: If you intend to change the type of transformation for an already existing / running workflow, you must also drop the target table(s) before changing the transformation type. The transformation types are mutually exclusive and cannot be applied to the same target table(s).","defaultValue":[{"values":{"1":"Copy Table"}}],"defaultValueType":"TEXT","required":true},{"slot":12,"variableName":"append_metadata","variableType":"SCALAR","displayName":"Append All Metadata","description":"Append all metadata columns to the target table. Options are:\n\nY: Append all metadata columns\nN: Only append metadata columns required by this job","defaultValue":[{"values":{"1":"Y"}}],"defaultValueType":"TEXT","required":true},{"slot":13,"variableName":"azure_key_vault_url","variableType":"SCALAR","displayName":"Azure Key Vault URL","description":"Required for tagging files in Azure Blob Storage\nThe URL of the Azure Key Vault used to store the Azure Blob Storage connection string. This is also called the vault's \"DNS Name\".","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":false},{"slot":14,"variableName":"storage_account_key_secret","variableType":"SCALAR","displayName":"Azure Blob Storage Account Key Secret","description":"Required for tagging files in Azure Blob Storage\nName of the Azure Key Vault secret which stores the Azure Blob Storage account key.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":false},{"slot":15,"variableName":"schema_drift_action","variableType":"SCALAR","displayName":"Schema Drift Action","description":"The action taken when schema changes are detected in the source table, which require an update to the target table:\n\nUpdate Target: The target table is updated according to the new schema.\nFail Job: This job will fail, causing no changes to the target table.","defaultValue":[{"values":{"1":"Update Target"}}],"defaultValueType":"TEXT","required":true},{"slot":16,"variableName":"logging_enabled","variableType":"SCALAR","displayName":"Logging Enabled","description":"Enable logging of row counts from transformation jobs. Options are:\n\nY: Log row counts\nN: Do not log row counts\n\nNote: Logged row counts may not match the number of rows processed by the CDC agent. For the Copy Table and Copy Table with Soft Deletes transformation types, if a given record has more than one change event in a job execution, only the final change event will be processed. The \"after\" object of the final change event contains the net changes from all the previous events.","defaultValue":[{"values":{"1":"Y"}}],"defaultValueType":"TEXT","required":true},{"slot":17,"variableName":"log_table","variableType":"SCALAR","displayName":"Log Table","description":"The name of the table in Snowflake containing the logged events.","defaultValue":[{"values":{"1":"MTLN_CDC_LOG"}}],"defaultValueType":"TEXT","required":true},{"slot":18,"variableName":"trim_all_strings","variableType":"SCALAR","displayName":"Trim All String Columns","description":"Remove trailing whitespace from string columns. Options are:\n\nY: Remove trailing whitespace\nN: Do not remove trailing whitespace\n","defaultValue":[{"values":{"1":"N"}}],"defaultValueType":"TEXT","required":true},{"slot":19,"variableName":"tag_processed_files","variableType":"SCALAR","displayName":"Tag Processed Files","description":"Once the Avro files have been processed, update the tags or metadata in cloud storage. This enables a lifecycle management policy to be applied to files to reduce storage costs. Options are:\n\nY: Update tags / metadata\nN: Do not update tags / metadata\n\nS3 and Azure Blob Storage\nA tag of matillion_cdc_processed = true will be added to each file. If a file already has the maximum 10 tags, the tagging process will be skipped.\n\nNote: For Azure Blob Storage, Blob Index Tags are not supported on Storage Accounts with a hierarchical namespace enabled.\n\nGoogle Cloud Storage\nThe 'Custom-Time' metadata will be updated to be the time the file was processed. If the 'Custom-Time' has already been set to a value in the future, the metadata update will be skipped.","defaultValue":[{"values":{"1":"N"}}],"defaultValueType":"TEXT","required":true},{"slot":20,"variableName":"bytes_to_decimal_function","variableType":"SCALAR","displayName":"Bytes to Decimal Function","description":"For some source databases (e.g. PostgreSQL and Oracle), numeric columns can be defined without specifying a scale or precision, and these columns are able to contain values with different scales and precisions. When these columns are extracted to the Avro files, they are denoted as the type VariableScaleDecimal.\n\nA VariableScaleDecimal value is a variant object with two properties, the value stored as a byte array, and a numeric scale. Snowflake SQL cannot natively convert these byte arrays back to the original numeric format. If your source tables contain such columns, a User Defined Function is required by the shared job to convert the values.\n\nIf no value is specified for this parameter, the shared job will not attempt to convert the byte arrays, and will load them to columns of type variant.\n\nIf a value is specified, the shared job will check whether the user defined function already exists. If it does not exist, the job will attempt to create it. If the job cannot create the function due to insufficient privileges, the job will fail and the function must be created in the Snowflake Console.\n\nIf the parameter is set to a value like BYTES_TO_DECIMAL, the job will look for the function in the environment default database and schema.\n\nIf the parameter is set to a value like CDC_DB.PUBLIC.BYTES_TO_DECIMAL, the job will look for the function in the PUBLIC schema of the CDC_DB database.","defaultValue":[{"values":{"1":"BYTES_TO_DECIMAL"}}],"defaultValueType":"TEXT","required":false},{"slot":21,"variableName":"use_source_schemas","variableType":"SCALAR","displayName":"Use Source Schemas","description":"Optionally create the target tables in a schema with the same name as the source table.\nIf the schema doesn't already exist, the job will try to create it.\n\nY: Create the target tables in a schema with the same name as the source.\nN: Create the target tables in the schema specified by the Target Schema parameter.\n\nNote: Which ever option is selected, the external tables will still be created in the schema specified by the Target Schema parameter.","defaultValue":[{"values":{"1":"N"}}],"defaultValueType":"TEXT","required":false},{"slot":22,"variableName":"primary_key_override","variableType":"GRID","displayName":"Primary Key Override","description":"Optionally provide a list of primary key columns for the source tables.\n\nBy default, the job will read the primary key columns from the CDC Avro files. However, if the source table does not a have a primary defined in its DDL, a list of unique columns can be specified here to enable Copy Table transformations.\n\nNote: The values for the source_database, source_schema, source_table, and source_column are case sensitive, and must match the source database.","defaultValue":[{"values":{"1":""}}],"defaultValueType":null,"required":false}]},"orchestrationJobs":["Sync All Tables - 1 - Iterate over tables","Sync All Tables - 1-2 - Identify connector","Sync All Tables - 1-1 - Get source tables"],"transformationJobs":[],"imageMetadata":{"componentIcon":"iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAACCElEQVR42mNgGAU4gJKSUjsQ/wbi/3jwdVlZWSlaOeC3srLycSDdgQVvgTrgDxDfpIkjoBZ0YJMDOiwBJK+oqJgCpD/TxBHEOEBBQUEDSNvQxBHEOgCqlvqOIMUB6I7Q0NAQprUDPKHyL4D4MRL+ChJXUVFxoKkDgIARKFcIxPPR8E6QPqADDwDp1SD9wFDioIUDcOlxwVJWFA6EA5yBPheAhkYP3R0ASgPa2to8ow4YdcBgcMBvujoAaJkxsIYsBjqAL22mMStQiBmUHenmABgIXaUkF7RccQ4ti2I8litIBC5TeATE7wKWKqwIXKoY57DfgYVcB8wBBqsOIQwMcna4xv8MjEDL24H4UsAyxTAgvReID7gtEucm1QFnCbQHYfgryAHeS+UEgb6dAtMfsFzOGUQ3NDAwBS5XWAV0DGmJUU5OThDouwigBfOgza8qII7Bgv1A6oOWy0cFLFPYgc2s4GWKesBQeEpuWsiANj4M8KkLXK6YCrRkJa5ECZT7QK4DEqFBvRtax6Pj5Q4ODiyBK+Q1A5fJvw5dJSME0geKEljiC1oq7wiMhg1kOUBGRkYIaMl5AunABxwKyxR7A5cqnApeLm8YuEzJBujrjQnzIQ2S0FXaPBRlS5AvcWGgNBMsBwCzXibQ4ptA/A2I/wctV5g5ID0raLAfh0UJ3QGoQKI42EcEAACvnDRCcfu4cgAAAABJRU5ErkJggg==","smallIcon":"iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAABD0lEQVR42mNgoCZQUlK6CMSHQVhRUfEckP4CxM6kGHAYxlZQUDAA8hcDDdpPtCHoBigrK08GivGDDAGynUgyQEZGRgjI3wfEB4H4DNCAA0CDfIFSTEQZgAzU1NSAUkoXQPJAl9mTa8B8IG4DusJt4AwAhkMLWQaoq6vz6loqaAQuktd0cHBgwWfACaAtYsgYpBkkF7BUISFwucKGwGUKFaGrGJhxGdALxM+BeCsQr4XiIKDGPJC891I5wYClimYByxRj8LliE9DmNKBfw0BYRUVFG2hAR9pMY1YQDl2lIAHkZ+A0ABjPFkBDGoGam6A4xG+xskrAMoV+IN4RvExBA6cXCIGg5fLeDvvxBCI5AAAP0VqipKOQ1AAAAABJRU5ErkJggg=="},"description":"Please see the full shared job documentation at the link - https://documentation.matillion.com/docs/sharedjobs-cdc","helpHtml":"

Sync All Tables

Please see the full shared job documentation at the link - https://documentation.matillion.com/docs/sharedjobs-cdc

Properties

PropertyTypeDescription
Cloud Storage URLVariableThe URL of the location where the CDC change event files are stored. This should have the format:\n\ns3:////\nor\nazure://.blob.core.windows.net///\nor\ngs:////
WarehouseVariableThe Snowflake warehouse used to process the data.
Target DatabaseVariableThe Snowflake database containing the external table with change events and the target table.
Target SchemaVariableThe Snowflake schema containing the external table with change events and the target table.
Snowflake External StageVariableThe name of the existing external stage which contains the files output by the CDC pipeline.
External TableVariableThe external table in Snowflake containing the CDC change events.
ConcurrencyVariableHow to iterate over the source tables for which there are CDC event data:\n\nSequential: Iterations are done in sequence, waiting for each to complete before starting the next. This is the default.\nConcurrent: Iterations are run concurrently.
Target Table PrefixVariableA prefix to prepend to the source table name to generate the target table name. If no Target Table Prefix is specified, the target table will have the same name as the source table.
Fully Qualify Target Table (Recommended)VariableOptionally includes the source database and schema in the target table name.\n\nY: The target table name will have the format \n__\n\nN: The target table name will have the format \n\n\nIt is strongly recommended to leave this set to 'Y', unless you are confident that your source table names will always be unique.\n\nIf the target table name is not fully qualified, and there are source tables with the same name in different databases or schemas, this job will see them as the same table and attempt to merge their metadata and contents.
Transformation TypeVariableThe type of transformation used when applying the change events to the target table.\n\nCopy Table: The target table will be maintained as a copy of the source table\nCopy Table with Soft Deletes: Same as Copy Table, but records deleted in the source table will be retained in the target table.\nChange Log: All change events will be extracted and appended to the target table\n\nNote: A primary key is required on the source table for Copy Table and Copy Table with Soft Deletes transformations. The primary key is used by the shared job to merge updates into the target table. If the source table does not have a primary key, the transformation type will be updated to Change Log.\n\nWarning: If you intend to change the type of transformation for an already existing / running workflow, you must also drop the target table(s) before changing the transformation type. The transformation types are mutually exclusive and cannot be applied to the same target table(s).
Append All MetadataVariableAppend all metadata columns to the target table. Options are:\n\nY: Append all metadata columns\nN: Only append metadata columns required by this job
Azure Key Vault URLVariableRequired for tagging files in Azure Blob Storage\nThe URL of the Azure Key Vault used to store the Azure Blob Storage connection string. This is also called the vault's \"DNS Name\".
Azure Blob Storage Account Key SecretVariableRequired for tagging files in Azure Blob Storage\nName of the Azure Key Vault secret which stores the Azure Blob Storage account key.
Schema Drift ActionVariableThe action taken when schema changes are detected in the source table, which require an update to the target table:\n\nUpdate Target: The target table is updated according to the new schema.\nFail Job: This job will fail, causing no changes to the target table.
Logging EnabledVariableEnable logging of row counts from transformation jobs. Options are:\n\nY: Log row counts\nN: Do not log row counts\n\nNote: Logged row counts may not match the number of rows processed by the CDC agent. For the Copy Table and Copy Table with Soft Deletes transformation types, if a given record has more than one change event in a job execution, only the final change event will be processed. The \"after\" object of the final change event contains the net changes from all the previous events.
Log TableVariableThe name of the table in Snowflake containing the logged events.
Trim All String ColumnsVariableRemove trailing whitespace from string columns. Options are:\n\nY: Remove trailing whitespace\nN: Do not remove trailing whitespace\n
Tag Processed FilesVariableOnce the Avro files have been processed, update the tags or metadata in cloud storage. This enables a lifecycle management policy to be applied to files to reduce storage costs. Options are:\n\nY: Update tags / metadata\nN: Do not update tags / metadata\n\nS3 and Azure Blob Storage\nA tag of matillion_cdc_processed = true will be added to each file. If a file already has the maximum 10 tags, the tagging process will be skipped.\n\nNote: For Azure Blob Storage, Blob Index Tags are not supported on Storage Accounts with a hierarchical namespace enabled.\n\nGoogle Cloud Storage\nThe 'Custom-Time' metadata will be updated to be the time the file was processed. If the 'Custom-Time' has already been set to a value in the future, the metadata update will be skipped.
Bytes to Decimal FunctionVariableFor some source databases (e.g. PostgreSQL and Oracle), numeric columns can be defined without specifying a scale or precision, and these columns are able to contain values with different scales and precisions. When these columns are extracted to the Avro files, they are denoted as the type VariableScaleDecimal.\n\nA VariableScaleDecimal value is a variant object with two properties, the value stored as a byte array, and a numeric scale. Snowflake SQL cannot natively convert these byte arrays back to the original numeric format. If your source tables contain such columns, a User Defined Function is required by the shared job to convert the values.\n\nIf no value is specified for this parameter, the shared job will not attempt to convert the byte arrays, and will load them to columns of type variant.\n\nIf a value is specified, the shared job will check whether the user defined function already exists. If it does not exist, the job will attempt to create it. If the job cannot create the function due to insufficient privileges, the job will fail and the function must be created in the Snowflake Console.\n\nIf the parameter is set to a value like BYTES_TO_DECIMAL, the job will look for the function in the environment default database and schema.\n\nIf the parameter is set to a value like CDC_DB.PUBLIC.BYTES_TO_DECIMAL, the job will look for the function in the PUBLIC schema of the CDC_DB database.
Use Source SchemasVariableOptionally create the target tables in a schema with the same name as the source table.\nIf the schema doesn't already exist, the job will try to create it.\n\nY: Create the target tables in a schema with the same name as the source.\nN: Create the target tables in the schema specified by the Target Schema parameter.\n\nNote: Which ever option is selected, the external tables will still be created in the schema specified by the Target Schema parameter.
Primary Key OverrideGridOptionally provide a list of primary key columns for the source tables.\n\nBy default, the job will read the primary key columns from the CDC Avro files. However, if the source table does not a have a primary defined in its DDL, a list of unique columns can be specified here to enable Copy Table transformations.\n\nNote: The values for the source_database, source_schema, source_table, and source_column are case sensitive, and must match the source database.
","created":1709736840591},"orchestrationJobs":{"Sync All Tables - 1 - Iterate over tables":{"id":-2,"revision":17,"created":1709736813392,"timestamp":1709736813392,"components":{"44035":{"id":44035,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1299670411,"x":-640,"y":0,"width":32,"height":32,"inputConnectorIDs":[44036],"outputSuccessConnectorIDs":[45891],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Create or refresh external table"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Target Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Target Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"External Snowflake Stage","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${snowflake_stage}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"External Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${external_table}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Cloud Storage URL","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${cloud_storage_url}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44364":{"id":44364,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":-1120,"y":0,"width":32,"height":32,"inputConnectorIDs":[50214],"outputSuccessConnectorIDs":[45888],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Update [Environment Default] values"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"if warehouse == '[Environment Default]':\n print(f'Updating warehouse from [Environment Default] to: {environment_port}')\n context.updateVariable('warehouse', environment_port)\n\nif target_database == '[Environment Default]':\n print(f'Updating target_database from [Environment Default] to: {environment_database}')\n context.updateVariable('target_database', environment_database)\n\nif target_schema == '[Environment Default]':\n print(f'Updating target_schema from [Environment Default] to: {environment_default_schema}')\n context.updateVariable('target_schema', environment_default_schema)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44366":{"id":44366,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":-880,"y":0,"width":32,"height":32,"inputConnectorIDs":[45888],"outputSuccessConnectorIDs":[44036],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Build external table name"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"external_table = external_table.upper()\n\nprint(f'external_table: {external_table}')\ncontext.updateVariable('external_table', external_table)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44367":{"id":44367,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":-1520,"y":0,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[50211],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Start","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44368":{"id":44368,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"ITERATE","implementationID":-424773870,"x":320,"y":0,"width":32,"height":16,"inputConnectorIDs":[44378],"outputSuccessConnectorIDs":[44376],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Iterate over table_list"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Grid Variable","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"source_tables"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Grid Variable Mapping","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"source_database"},"2":{"slot":2,"type":"STRING","value":"source_database"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"source_schema"},"2":{"slot":2,"type":"STRING","value":"source_schema"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"source_table"},"2":{"slot":2,"type":"STRING","value":"source_table"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Break on Failure","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"No"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Concurrency","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${concurrency}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Stop on Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"No"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":false,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Condition","elements":{},"visible":false,"status":{},"columnNames":[]},"10":{"slot":10,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":false,"status":{},"columnNames":[]},"11":{"slot":11,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"999":{"slot":999,"name":"Record Values In Task History","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Yes"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[56551],"inputIterationConnectorIDs":[]},"44369":{"id":44369,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":480,"y":0,"width":32,"height":32,"inputConnectorIDs":[44376],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Success"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44370":{"id":44370,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":-400,"y":0,"width":32,"height":32,"inputConnectorIDs":[45891],"outputSuccessConnectorIDs":[44395],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"source_table_count","mapTo":"source_table_count","validationStatus":null,"validationMessage":null}},"gridExportMappings":{"1":{"slot":1,"fromGrid":"source_tables","toGrid":"source_tables","columnMapToFrom":{"source_table":"source_table","source_schema":"source_schema","source_database":"source_database"},"validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Get list of source tables in pipeline"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync All Tables - 1-1 - Get source tables"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"target_database"},"2":{"slot":2,"type":"STRING","value":"${target_database}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"target_schema"},"2":{"slot":2,"type":"STRING","value":"${target_schema}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"external_table"},"2":{"slot":2,"type":"STRING","value":"${external_table}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync All Tables (Exchange)"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Data Loader.CDC"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"3"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44371":{"id":44371,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":80,"y":0,"width":32,"height":32,"inputConnectorIDs":[44398],"outputSuccessConnectorIDs":[44378],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"connector","mapTo":"connector","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Identify connector for pipeline"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync All Tables - 1-2 - Identify connector"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"target_database"},"2":{"slot":2,"type":"STRING","value":"${target_database}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"target_schema"},"2":{"slot":2,"type":"STRING","value":"${target_schema}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"external_table"},"2":{"slot":2,"type":"STRING","value":"${external_table}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync All Tables (Exchange)"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Data Loader.CDC"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"3"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44387":{"id":44387,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":-160,"y":0,"width":32,"height":32,"inputConnectorIDs":[44395],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[44398],"outputFalseConnectorIDs":[44389],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Source tables found in cloud storage?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"source_table_count"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Greater than"},"4":{"slot":4,"type":"STRING","value":"0"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44388":{"id":44388,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":-160,"y":160,"width":32,"height":32,"inputConnectorIDs":[44389],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Success - No files in cloud storage"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"50162":{"id":50162,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":-1360,"y":0,"width":32,"height":32,"inputConnectorIDs":[50211],"outputSuccessConnectorIDs":[50214],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Print version"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"print('Version: 3.1.1')"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"56586":{"id":56586,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":936797420,"x":320,"y":32,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync single table"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Data Source","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${connector}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Source Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${source_database}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Source Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${source_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Source Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${source_table}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Cloud Storage URL","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${cloud_storage_url}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Warehouse","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${warehouse}"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Target Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Target Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"10":{"slot":10,"name":"External Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${external_table}"}}}},"visible":true,"status":{},"columnNames":[]},"11":{"slot":11,"name":"Target Table Prefix","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_prefix}"}}}},"visible":true,"status":{},"columnNames":[]},"12":{"slot":12,"name":"Fully Qualify Target Table (Recommended)","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${fully_qualify_target_table}"}}}},"visible":true,"status":{},"columnNames":[]},"13":{"slot":13,"name":"Transformation Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${transformation_type}"}}}},"visible":true,"status":{},"columnNames":[]},"14":{"slot":14,"name":"Append All Metadata","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${append_metadata}"}}}},"visible":true,"status":{},"columnNames":[]},"15":{"slot":15,"name":"Azure Key Vault URL","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${azure_key_vault_url}"}}}},"visible":true,"status":{},"columnNames":[]},"16":{"slot":16,"name":"Azure Blob Storage Account Key Secret","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${storage_account_key_secret}"}}}},"visible":true,"status":{},"columnNames":[]},"17":{"slot":17,"name":"Schema Drift Action","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${schema_drift_action}"}}}},"visible":true,"status":{},"columnNames":[]},"18":{"slot":18,"name":"Logging Enabled","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${logging_enabled}"}}}},"visible":true,"status":{},"columnNames":[]},"19":{"slot":19,"name":"Log Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${log_table}"}}}},"visible":true,"status":{},"columnNames":[]},"20":{"slot":20,"name":"Trim All String Columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${trim_all_strings}"}}}},"visible":true,"status":{},"columnNames":[]},"21":{"slot":21,"name":"Tag Processed Files","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${tag_processed_files}"}}}},"visible":true,"status":{},"columnNames":[]},"22":{"slot":22,"name":"Bytes to Decimal Function","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${bytes_to_decimal_function}"}}}},"visible":true,"status":{},"columnNames":[]},"23":{"slot":23,"name":"Use Source Schemas","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${use_source_schemas}"}}}},"visible":true,"status":{},"columnNames":[]},"24":{"slot":24,"name":"Primary Key Override","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"primary_key_override"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"source_database"},"2":{"slot":2,"type":"GRID","value":"source_schema"},"3":{"slot":3,"type":"GRID","value":"source_table"},"4":{"slot":4,"type":"GRID","value":"source_column"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[56551]}},"successConnectors":{"44036":{"id":44036,"sourceID":44366,"targetID":44035},"44376":{"id":44376,"sourceID":44368,"targetID":44369},"44378":{"id":44378,"sourceID":44371,"targetID":44368},"44395":{"id":44395,"sourceID":44370,"targetID":44387},"45888":{"id":45888,"sourceID":44364,"targetID":44366},"45891":{"id":45891,"sourceID":44035,"targetID":44370},"50214":{"id":50214,"sourceID":50162,"targetID":44364}},"failureConnectors":{},"unconditionalConnectors":{"50211":{"id":50211,"sourceID":44367,"targetID":50162}},"trueConnectors":{"44398":{"id":44398,"sourceID":44387,"targetID":44371}},"falseConnectors":{"44389":{"id":44389,"sourceID":44387,"targetID":44388}},"iterationConnectors":{"56551":{"id":56551,"sourceID":44368,"targetID":56586}},"noteConnectors":{},"canUndo":true,"undoCommand":"Edit Note","undoCreated":1709736766062,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"44356":{"id":44356,"x":-715,"y":-153,"width":150,"height":229,"text":"**Create or refresh external table**\n\nCreate external table, if it doesn't exist.\n\nOtherwise, refresh the external table with the latest files.","colour":"e6e63c"},"44357":{"id":44357,"x":-1195,"y":-154,"width":150,"height":228,"text":"**Update [Environment Default] values**\n\nReplace any [Environment Default] parameters with the actual value","colour":"e6e63c"},"44358":{"id":44358,"x":-2233,"y":-393,"width":631,"height":147,"text":"**Sync CDC event data from all source tables in an external table**\n\n- Get list of all source tables with event data in cloud storage\n- Refresh external table with latest files\n- Iterate over table list, syncing each table in turn","colour":"00ce4f"},"44359":{"id":44359,"x":-955,"y":-154,"width":151,"height":228,"text":"**Build table names**\n\nEnsure external table name is upper case.","colour":"e6e63c"},"44360":{"id":44360,"x":245,"y":-157,"width":149,"height":229,"text":"**Load new data**\n\nIterate over source table list, loading new data","colour":"e6e63c"},"44361":{"id":44361,"x":-2233,"y":-154,"width":626,"height":629,"text":"**Change log**\n\n1.0.0 -----------------------------------------------------------------------------------------------------------\n- General availability\n- Support for PostgreSQL, Oracle and SQL Server sources\n- Support for S3 and Azure Blob Storage locations\n- Support for Copy Table, Copy Table With Soft Deletes and Change Log transformation types\n- Support for schema drift\n\n1.1.0 -----------------------------------------------------------------------------------------------------------\n- Support for Google Cloud Storage locations\n\n2.0.0 -----------------------------------------------------------------------------------------------------------\n- Call 2.0.0 versions of Create or Refresh External Table and Sync Single Table\n\n2.1.0 -----------------------------------------------------------------------------------------------------------\n- Call 2.1.0 version of Sync Single Table, with logging parameters\n\n2.2.0 -----------------------------------------------------------------------------------------------------------\n- Call 2.2.0 version of Sync Single Table, with new parameter Trim All String Columns\n\n2.3.0 -----------------------------------------------------------------------------------------------------------\n- Optionally add tag to each Avro file after the data has been loaded\n\n2.4.0 -----------------------------------------------------------------------------------------------------------\n- Added parameter for User Defined Function to convert byte arrays to decimals\n\n3.0.0 -----------------------------------------------------------------------------------------------------------\n- For postgresql, switched to using metadata.sequence as source for MTLN_CDC_SEQUENCE_NUMBER, changing data type from NUMBER to VARCHAR. Existing pipelines will be converted to the new format.\n- For oracle, switched to using metadata.rs_id and metadata.ssn as source for MTLN_CDC_SEQUENCE_NUMBER, changing data type from NUMBER to VARCHAR. Existing pipelines must be cleared down and re-snapshot.\n\n3.1.0 -----------------------------------------------------------------------------------------------------------\n- Added optional Primary Key Override parameter so that a primary key can be used for Copy Table transformations when the source table does not have the primary key defined in DDL\n- Added optional Use Source Schemas parameter to create target tables in schemas with the same name as the source. The Target Schema is still used to contain the external tables\n- Query the EXTERNAL_TABLE_FILES function to obtain a list of all source tables in this CDC pipeline\n\n3.1.1 -----------------------------------------------------------------------------------------------------------\n- If no source tables found, end job before attempting to read connector\n","colour":"00ce4f"},"44362":{"id":44362,"x":-474,"y":-154,"width":150,"height":230,"text":"**Get list of source tables**\n\nQuery cloud storage to obtain a list of all source tables in this CDC pipeline","colour":"e6e63c"},"44363":{"id":44363,"x":6,"y":-157,"width":148,"height":230,"text":"**Identify connector for pipeline**","colour":"e6e63c"},"44440":{"id":44440,"x":-234,"y":-155,"width":147,"height":230,"text":"**Check we found source tables**\n\nIf no source tables found in cloud storage, end job now before attempting to read the connector.","colour":"e6e63c"},"50163":{"id":50163,"x":-1437,"y":-153,"width":152,"height":229,"text":"**Print version**\n\nPrint shared job version to the task history, to help support cases","colour":"e6e63c"}},"variables":{"connector":{"definition":{"name":"connector","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"log_table":{"definition":{"name":"log_table","type":"TEXT","scope":"TASKBATCH","description":"The name of the table in Snowflake containing the logged events.","visibility":"PUBLIC"},"value":""},"warehouse":{"definition":{"name":"warehouse","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"concurrency":{"definition":{"name":"concurrency","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"Sequential"},"source_table":{"definition":{"name":"source_table","type":"TEXT","scope":"BRANCH","description":"","visibility":"PRIVATE"},"value":""},"source_schema":{"definition":{"name":"source_schema","type":"TEXT","scope":"BRANCH","description":"","visibility":"PRIVATE"},"value":""},"target_prefix":{"definition":{"name":"target_prefix","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_schema":{"definition":{"name":"target_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"external_table":{"definition":{"name":"external_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"append_metadata":{"definition":{"name":"append_metadata","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"Y"},"logging_enabled":{"definition":{"name":"logging_enabled","type":"TEXT","scope":"TASKBATCH","description":"Enable logging of row counts from transformation jobs. Options are:\n\nY: Log row counts\nN: Do not log row counts\n\nNote: Logged row counts may not match the number of rows processed by the CDC agent. For the Copy Table and Copy Table with Soft Deletes transformation types, if a given record has more than one change event in a job execution, only the final change event will be processed. The \"after\" object of the final change event contains the net changes from all the previous events.","visibility":"PUBLIC"},"value":""},"snowflake_stage":{"definition":{"name":"snowflake_stage","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"source_database":{"definition":{"name":"source_database","type":"TEXT","scope":"BRANCH","description":"","visibility":"PRIVATE"},"value":""},"target_database":{"definition":{"name":"target_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"trim_all_strings":{"definition":{"name":"trim_all_strings","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"Default"},"cloud_storage_url":{"definition":{"name":"cloud_storage_url","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"source_table_count":{"definition":{"name":"source_table_count","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"0"},"use_source_schemas":{"definition":{"name":"use_source_schemas","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"N"},"azure_key_vault_url":{"definition":{"name":"azure_key_vault_url","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"schema_drift_action":{"definition":{"name":"schema_drift_action","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"Update Target"},"tag_processed_files":{"definition":{"name":"tag_processed_files","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"N"},"transformation_type":{"definition":{"name":"transformation_type","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"Copy Table"},"bytes_to_decimal_function":{"definition":{"name":"bytes_to_decimal_function","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"BYTES_TO_DECIMAL"},"fully_qualify_target_table":{"definition":{"name":"fully_qualify_target_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"storage_account_key_secret":{"definition":{"name":"storage_account_key_secret","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""}},"grids":{"source_tables":{"definition":{"name":"source_tables","scope":"TASKBATCH","definitions":[{"name":"source_database","type":"TEXT"},{"name":"source_schema","type":"TEXT"},{"name":"source_table","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[]},"primary_key_override":{"definition":{"name":"primary_key_override","scope":"TASKBATCH","definitions":[{"name":"source_database","type":"TEXT"},{"name":"source_schema","type":"TEXT"},{"name":"source_table","type":"TEXT"},{"name":"source_column","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]}}},"Sync All Tables - 1-2 - Identify connector":{"id":-2,"revision":1,"created":1709542096647,"timestamp":1709542096647,"components":{"44342":{"id":44342,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":-480,"y":0,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[44354],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Start","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44343":{"id":44343,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":-320,"y":0,"width":32,"height":32,"inputConnectorIDs":[44354],"outputSuccessConnectorIDs":[44349],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Build get_connector_sql"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"# read any single record from the stage table to find the connector\nget_connector_sql = \\\nf\"\"\"SELECT\n coalesce(any_value(\"CONNECTOR\"), '') AS \"CONNECTOR\"\nFROM (\n SELECT \n \"VALUE\":\"metadata\".\"connector\"::VARCHAR AS \"CONNECTOR\"\n FROM\n \"{target_database}\".\"{target_schema}\".\"{external_table}\"\n LIMIT 1\n)\"\"\"\n\nprint(f'get_connector_sql: \\n{get_connector_sql}')\ncontext.updateVariable('get_connector_sql', get_connector_sql)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44344":{"id":44344,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1032749985,"x":-160,"y":0,"width":32,"height":32,"inputConnectorIDs":[44349],"outputSuccessConnectorIDs":[44350],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Get source connector from stage table"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"SQL Query","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${get_connector_sql}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Scalar Variable Mapping","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"connector"},"2":{"slot":2,"type":"STRING","value":"CONNECTOR"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"[Environment Default]"}}}},"visible":false,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Table Columns","elements":{},"visible":false,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"[Environment Default]"}}}},"visible":false,"status":{},"columnNames":[]},"10":{"slot":10,"name":"Limit","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"100"}}}},"visible":false,"status":{},"columnNames":[]},"11":{"slot":11,"name":"Order By","elements":{},"visible":false,"status":{},"columnNames":[]},"12":{"slot":12,"name":"Sort","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Ascending"}}}},"visible":false,"status":{},"columnNames":[]},"13":{"slot":13,"name":"Filter Conditions","elements":{},"visible":false,"status":{},"columnNames":[]},"14":{"slot":14,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"AND"}}}},"visible":false,"status":{},"columnNames":[]},"20":{"slot":20,"name":"Basic / Advanced","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Advanced"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44345":{"id":44345,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":0,"y":0,"width":32,"height":32,"inputConnectorIDs":[44350],"outputSuccessConnectorIDs":[44351],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Print connector"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"print(f'connector: {connector!r}')\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44346":{"id":44346,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":160,"y":0,"width":32,"height":32,"inputConnectorIDs":[44351],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[44352],"outputFalseConnectorIDs":[44353],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Connector identified?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"connector"},"2":{"slot":2,"type":"STRING","value":"Not"},"3":{"slot":3,"type":"STRING","value":"Blank"},"4":{"slot":4,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44347":{"id":44347,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":320,"y":0,"width":32,"height":32,"inputConnectorIDs":[44352],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Success"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44348":{"id":44348,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":158,"y":160,"width":32,"height":32,"inputConnectorIDs":[44353],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Failure - No connector identified for pipeline"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"44349":{"id":44349,"sourceID":44343,"targetID":44344},"44350":{"id":44350,"sourceID":44344,"targetID":44345},"44351":{"id":44351,"sourceID":44345,"targetID":44346}},"failureConnectors":{},"unconditionalConnectors":{"44354":{"id":44354,"sourceID":44342,"targetID":44343}},"trueConnectors":{"44352":{"id":44352,"sourceID":44346,"targetID":44347}},"falseConnectors":{"44353":{"id":44353,"sourceID":44346,"targetID":44348}},"iterationConnectors":{},"noteConnectors":{},"canUndo":false,"undoCommand":"","undoCreated":0,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"44341":{"id":44341,"x":-395,"y":-155,"width":629,"height":231,"text":"**Check source connector**\n\nQuery the external table to retrieve the connector from any single change event","colour":"e6e63c"}},"variables":{"connector":{"definition":{"name":"connector","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_schema":{"definition":{"name":"target_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"external_table":{"definition":{"name":"external_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_database":{"definition":{"name":"target_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"get_connector_sql":{"definition":{"name":"get_connector_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""}},"grids":{}},"Sync All Tables - 1-1 - Get source tables":{"id":-2,"revision":7,"created":1709736690981,"timestamp":1709736690981,"components":{"44325":{"id":44325,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":-80,"y":160,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[45695],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44326":{"id":44326,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":560,"y":160,"width":32,"height":32,"inputConnectorIDs":[44434],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Success"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"45694":{"id":45694,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":80,"y":160,"width":32,"height":32,"inputConnectorIDs":[45695],"outputSuccessConnectorIDs":[45722],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Build list_tables_sql"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"list_tables_sql = \\\nf'''SELECT \n SPLIT_PART(SPLIT_PART(\"FILE_NAME\", '/', -9), '=', -1) AS \"DATABASE\",\n SPLIT_PART(SPLIT_PART(\"FILE_NAME\", '/', -8), '=', -1) AS \"SCHEMA\",\n SPLIT_PART(SPLIT_PART(\"FILE_NAME\", '/', -7), '=', -1) AS \"TABLE\"\nFROM \n TABLE(\"{target_database}\".\"INFORMATION_SCHEMA\".\"EXTERNAL_TABLE_FILES\"(table_name=>'\"{target_database}\".\"{target_schema}\".\"{external_table}\"'))\nGROUP BY ALL\nORDER BY\n \"DATABASE\",\n \"SCHEMA\",\n \"TABLE\"'''\n\nprint(f'list_tables_sql: \\n{list_tables_sql}')\ncontext.updateVariable('list_tables_sql', list_tables_sql)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"45721":{"id":45721,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":1155583855,"x":240,"y":160,"width":32,"height":32,"inputConnectorIDs":[45722],"outputSuccessConnectorIDs":[45768],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Get list of source tables in pipeline"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Basic / Advanced","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Advanced"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"[Environment Default]"}}}},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Table Columns","elements":{},"visible":false,"status":{},"columnNames":[]},"7":{"slot":7,"name":"SQL Query","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${list_tables_sql}"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Grid Variable","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"source_tables"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Grid Variable Mapping","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DATABASE"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"SCHEMA"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"TABLE"}}}},"visible":true,"status":{},"columnNames":[]},"10":{"slot":10,"name":"Limit","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"100"}}}},"visible":false,"status":{},"columnNames":[]},"11":{"slot":11,"name":"Order By","elements":{},"visible":false,"status":{},"columnNames":[]},"12":{"slot":12,"name":"Sort","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Ascending"}}}},"visible":false,"status":{},"columnNames":[]},"13":{"slot":13,"name":"Filter Conditions","elements":{},"visible":false,"status":{},"columnNames":[]},"14":{"slot":14,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"AND"}}}},"visible":false,"status":{},"columnNames":[]},"20":{"slot":20,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"[Environment Default]"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"45767":{"id":45767,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":400,"y":160,"width":32,"height":32,"inputConnectorIDs":[45768],"outputSuccessConnectorIDs":[44434],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Count source tables found"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"source_tables = context.getGridVariable('source_tables') or []\n\nsource_table_count = len(source_tables)\n\nprint(f'source_table_count: {source_table_count}')\ncontext.updateVariable('source_table_count', source_table_count)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"44434":{"id":44434,"sourceID":45767,"targetID":44326},"45722":{"id":45722,"sourceID":45694,"targetID":45721},"45768":{"id":45768,"sourceID":45721,"targetID":45767}},"failureConnectors":{},"unconditionalConnectors":{"45695":{"id":45695,"sourceID":44325,"targetID":45694}},"trueConnectors":{},"falseConnectors":{},"iterationConnectors":{},"noteConnectors":{},"canUndo":true,"undoCommand":"Move Components","undoCreated":1709736689723,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"45840":{"id":45840,"x":5,"y":6,"width":472,"height":230,"text":"**Get list of source tables**\n\nQuery the EXTERNAL_TABLE_FILES function to obtain a list of all source tables in this CDC pipeline","colour":"e6e63c"}},"variables":{"target_schema":{"definition":{"name":"target_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"external_table":{"definition":{"name":"external_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"list_tables_sql":{"definition":{"name":"list_tables_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"target_database":{"definition":{"name":"target_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"source_table_count":{"definition":{"name":"source_table_count","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"}},"grids":{"source_tables":{"definition":{"name":"source_tables","scope":"TASKBATCH","definitions":[{"name":"source_database","type":"TEXT"},{"name":"source_schema","type":"TEXT"},{"name":"source_table","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]}}}},"transformationJobs":{}},{"metadata":{"identifier":{"packageName":"Matillion.Data Loader.CDC","name":"Sync Single Table (Exchange)","revision":3,"type":"DYNAMIC"},"rootJobReference":{"name":"Sync Single Table - 1 - Orchestrate","type":"ORCHESTRATION","parameterMetadata":[{"slot":2,"variableName":"connector","variableType":"SCALAR","displayName":"Data Source","description":"The CDC pipeline data source. Must be one of the supported CDC pipeline sources in Matillion Data Loader:\n\n- DB2_IBM_i\n- MySQL\n- Oracle\n- PosgtreSQL\n- SQLServer","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":3,"variableName":"source_database","variableType":"SCALAR","displayName":"Source Database","description":"The name of the database containing the source table.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":4,"variableName":"source_schema","variableType":"SCALAR","displayName":"Source Schema","description":"The name of the schema containing the source table.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":5,"variableName":"source_table","variableType":"SCALAR","displayName":"Source Table","description":"The name of the source table.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":6,"variableName":"cloud_storage_url","variableType":"SCALAR","displayName":"Cloud Storage URL","description":"The URL of the location where the CDC change event files are stored. This should have the format:\n\ns3:////\nor\nazure://.blob.core.windows.net///\nor\ngs:////","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":7,"variableName":"warehouse","variableType":"SCALAR","displayName":"Warehouse","description":"The Snowflake warehouse used to process the data.","defaultValue":[{"values":{"1":"[Environment Default]"}}],"defaultValueType":"TEXT","required":true},{"slot":8,"variableName":"target_database","variableType":"SCALAR","displayName":"Target Database","description":"The Snowflake database containing the external table with change events and the target table.","defaultValue":[{"values":{"1":"[Environment Default]"}}],"defaultValueType":"TEXT","required":true},{"slot":9,"variableName":"target_schema","variableType":"SCALAR","displayName":"Target Schema","description":"The Snowflake schema containing the external table with change events and the target table.","defaultValue":[{"values":{"1":"[Environment Default]"}}],"defaultValueType":"TEXT","required":true},{"slot":10,"variableName":"stage_table","variableType":"SCALAR","displayName":"External Table","description":"The external table in Snowflake containing the CDC change events.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":11,"variableName":"target_prefix","variableType":"SCALAR","displayName":"Target Table Prefix","description":"A prefix to prepend to the source table name to generate the target table name. If no Target Table Prefix is specified, the target table will have the same name as the source table.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":false},{"slot":12,"variableName":"fully_qualify_target_table","variableType":"SCALAR","displayName":"Fully Qualify Target Table (Recommended)","description":"Optionally includes the source database and schema in the target table name.\n\nY: The target table name will have the format \n__\n\nN: The target table name will have the format \n\n\nIt is strongly recommended to leave this set to 'Y', unless you are confident that your source table names will always be unique.\n\nIf the target table name is not fully qualified, and there are source tables with the same name in different databases or schemas, this job will see them as the same table and attempt to merge their metadata and contents.","defaultValue":[{"values":{"1":"Y"}}],"defaultValueType":"TEXT","required":true},{"slot":13,"variableName":"transformation_type","variableType":"SCALAR","displayName":"Transformation Type","description":"The type of transformation used when applying the change events to the target table.\n\nCopy Table: The target table will be maintained as a copy of the source table\nCopy Table with Soft Deletes: Same as Copy Table, but records deleted in the source table will be retained in the target table.\nChange Log: All change events will be extracted and appended to the target table\n\nNote: A primary key is required on the source table for Copy Table and Copy Table with Soft Deletes transformations. The primary key is used by the shared job to merge updates into the target table. If the source table does not have a primary key, the transformation type will be updated to Change Log.\n\nWarning: If you intend to change the type of transformation for an already existing / running workflow, you must also drop the target table(s) before changing the transformation type. The transformation types are mutually exclusive and cannot be applied to the same target table(s).","defaultValue":[{"values":{"1":"Copy Table"}}],"defaultValueType":"TEXT","required":true},{"slot":14,"variableName":"append_metadata","variableType":"SCALAR","displayName":"Append All Metadata","description":"Append all metadata columns to the target table. Options are:\n\nY: Append all metadata columns\nN: Only append metadata columns required by this job","defaultValue":[{"values":{"1":"Y"}}],"defaultValueType":"TEXT","required":true},{"slot":15,"variableName":"azure_key_vault_url","variableType":"SCALAR","displayName":"Azure Key Vault URL","description":"Required for tagging files in Azure Blob Storage\nThe URL of the Azure Key Vault used to store the Azure Blob Storage connection string. This is also called the vault's \"DNS Name\".","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":false},{"slot":16,"variableName":"storage_account_key_secret","variableType":"SCALAR","displayName":"Azure Blob Storage Account Key Secret","description":"Required for tagging files in Azure Blob Storage\nName of the Azure Key Vault secret which stores the Azure Blob Storage account key.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":false},{"slot":17,"variableName":"schema_drift_action","variableType":"SCALAR","displayName":"Schema Drift Action","description":"The action taken when schema changes are detected in the source table, which require an update to the target table:\n\nUpdate Target: The target table is updated according to the new schema.\nFail Job: This job will fail, causing no changes to the target table.","defaultValue":[{"values":{"1":"Update Target"}}],"defaultValueType":"TEXT","required":true},{"slot":18,"variableName":"logging_enabled","variableType":"SCALAR","displayName":"Logging Enabled","description":"Enable logging of row counts from transformation jobs. Options are:\n\nY: Log row counts\nN: Do not log row counts\n\nNote: Logged row counts may not match the number of rows processed by the CDC agent. For the Copy Table and Copy Table with Soft Deletes transformation types, if a given record has more than one change event in a job execution, only the final change event will be processed. The \"after\" object of the final change event contains the net changes from all the previous events.","defaultValue":[{"values":{"1":"Y"}}],"defaultValueType":"TEXT","required":true},{"slot":19,"variableName":"log_table","variableType":"SCALAR","displayName":"Log Table","description":"The name of the table in Snowflake containing the logged events.","defaultValue":[{"values":{"1":"MTLN_CDC_LOG"}}],"defaultValueType":"TEXT","required":true},{"slot":20,"variableName":"trim_all_strings","variableType":"SCALAR","displayName":"Trim All String Columns","description":"Remove trailing whitespace from string columns. Options are:\n\nY: Remove trailing whitespace\nN: Do not remove trailing whitespace","defaultValue":[{"values":{"1":"N"}}],"defaultValueType":"TEXT","required":true},{"slot":21,"variableName":"tag_processed_files","variableType":"SCALAR","displayName":"Tag Processed Files","description":"Once the Avro files have been processed, update the tags or metadata in cloud storage. This enables a lifecycle management policy to be applied to files to reduce storage costs. Options are:\n\nY: Update tags / metadata\nN: Do not update tags / metadata\n\nS3 and Azure Blob Storage\nA tag of matillion_cdc_processed = true will be added to each file. If a file already has the maximum 10 tags, the tagging process will be skipped.\n\nNote: For Azure Blob Storage, Blob Index Tags are not supported on Storage Accounts with a hierarchical namespace enabled.\n\nGoogle Cloud Storage\nThe 'Custom-Time' metadata will be updated to be the time the file was processed. If the 'Custom-Time' has already been set to a value in the future, the metadata update will be skipped.\n","defaultValue":[{"values":{"1":"N"}}],"defaultValueType":"TEXT","required":true},{"slot":22,"variableName":"bytes_to_decimal_function","variableType":"SCALAR","displayName":"Bytes to Decimal Function","description":"For some source databases (e.g. PostgreSQL and Oracle), numeric columns can be defined without specifying a scale or precision, and these columns are able to contain values with different scales and precisions. When these columns are extracted to the Avro files, they are denoted as the type VariableScaleDecimal.\n\nA VariableScaleDecimal value is a variant object with two properties, the value stored as a byte array, and a numeric scale. Snowflake SQL cannot natively convert these byte arrays back to the original numeric format. If your source tables contain such columns, a User Defined Function is required by the shared job to convert the values.\n\nIf no value is specified for this parameter, the shared job will not attempt to convert the byte arrays, and will load them to columns of type variant.\n\nIf a value is specified, the shared job will check whether the user defined function already exists. If it does not exist, the job will attempt to create it. If the job cannot create the function due to insufficient privileges, the job will fail and the function must be created in the Snowflake Console.\n\nIf the parameter is set to a value like BYTES_TO_DECIMAL, the job will look for the function in the environment default database and schema.\n\nIf the parameter is set to a value like CDC_DB.PUBLIC.BYTES_TO_DECIMAL, the job will look for the function in the PUBLIC schema of the CDC_DB database.","defaultValue":[{"values":{"1":"BYTES_TO_DECIMAL"}}],"defaultValueType":"TEXT","required":false},{"slot":23,"variableName":"use_source_schemas","variableType":"SCALAR","displayName":"Use Source Schemas","description":"Optionally create the target tables in a schema with the same name as the source table.\nIf the schema doesn't already exist, the job will try to create it.\n\nY: Create the target tables in a schema with the same name as the source.\nN: Create the target tables in the schema specified by the Target Schema parameter.\n\nNote: Which ever option is selected, the external tables will still be created in the schema specified by the Target Schema parameter.","defaultValue":[{"values":{"1":"N"}}],"defaultValueType":"TEXT","required":false},{"slot":24,"variableName":"primary_key_override","variableType":"GRID","displayName":"Primary Key Override","description":"Optionally provide a list of primary key columns for the source tables.\n\nBy default, the job will read the primary key columns from the CDC Avro files. However, if the source table does not a have a primary defined in its DDL, a list of unique columns can be specified here to enable Copy Table transformations.\n\nNote: The values for the source_database, source_schema, source_table, and source_column are case sensitive, and must match the source database.","defaultValue":[{"values":{"1":""}}],"defaultValueType":null,"required":false}]},"orchestrationJobs":["Sync Single Table - 1 - Orchestrate","Logging - 1 - Log Metrics","Sync Single Table - 1-2-2 - Validate event metadata","Sync Single Table - 1-2-5 - Transform and Load","Sync Single Table - 1-2-4 - Resolve metadata drift","Sync Single Table - 1-0 - Validate bytes to decimal function","Sync Single Table - 1-1 - Validate schema","Sync Single Table - 1-2-3 - Build new target table metadata","Sync Single Table - 1-2-6 - Apply tags","Sync Single Table - 1-2-1 - Get Avro schema","Sync Single Table - 1-2 - Process source metadata version","Sync Single Table - 1-2-4-1 - Alter postgresql sequence column"],"transformationJobs":["Logging - 1-1 - Append Metrics","Sync Single Table - 1-2-5-4 - Transform and load soft deletes","Sync Single Table - 1-2-5-2 - Transform and load","Sync Single Table - 1-2-5-1 - Transform and append","Sync Single Table - 1-2-5-3 - Transform and load without deletes"],"imageMetadata":{"componentIcon":"iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAABw0lEQVR42mNgGAU4gLKysqeSklI9EDfgw0B14VS3HGiwFxD/JxYrKiqWUNsBIJ//B/quDEi7YMNAS/dDLX9DdUdAgxfkOxc8apZD1dgA8X2qOoIUB6ioqMgoAAFVHUGqA0B8qjqCHAdQ1RFEOmApNKHWAC3LgWEgvwco/g+qX42WDogilD2BAWJAMwdA1ZkAfRyBBe+giwPw6J8y6oBRBwyoA4CFkzIQawEx+4A4IGCFrDLNi2JcIHC5YmrgMsUtDQ0MTHR3QPBSJdXAZQq/oPgDEC8PWqooTxcHuC0S5wbRQcvlvQOXya/2m6vOG7RUvgroiCehq5TkaOoA/+XyVoFLFXbC+L7L1EQQUaLQEbRcYT5NHQD0ZQXQkmnY5IJWyBsB5e/Q1AFByxRzA5YqLMUqt1TeEeiAi6Q2yWugjYqFBJrlRZAoUJYFWvIueJmcNjgxAunQVQoSYAcsUbAAOZAkBwAt1gMa/oWIJvk/oGNloYkvCuiIl8AEWB2wXLEGyL7ru0hekeysCCxGJYAWOONqlsMwSB08Gy5T0ACWAb3ABHkK6ID/QcvkiwekVxW8VN4LmCbyB6xb57DfgWW0c0ssAAAlCxuQVBzDUQAAAABJRU5ErkJggg==","smallIcon":"iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAA7ElEQVR42mNgoBaQkZERUlJS2qSoqLgfGQPFiogyQEVFxQGoeJ2ampoSDAPBTSA+CDSoiigDgAoXIosBNV9WVlYWA4pvJ2gIHgMcgeJ2QPYtIDuNJAOA/GSgxtVQfBiIp5NkAJprvAbWAKCcHjAMIsg2IHCpgj/JsQADoau0eQKXKewC4nYg9iDJAP8lSrpByxVD0mYas4JdskyxF6sBCgoK9sBA2gkyCIaBfJuAFbLKQcsUCtwWiXMDXcIWuFwhD6sB2traPEANU0CuQMbq6uq8gUvlowOXKk4BumQO2ZnNd5maSNASFRkGagMAz21T8LCeghAAAAAASUVORK5CYII="},"description":"Please see the full shared job documentation at the link - https://documentation.matillion.com/docs/sharedjobs-cdc","helpHtml":"

Sync Single Table

Please see the full shared job documentation at the link - https://documentation.matillion.com/docs/sharedjobs-cdc

Properties

PropertyTypeDescription
Data SourceVariableThe CDC pipeline data source. Must be one of the supported CDC pipeline sources in Matillion Data Loader:\n\n- DB2_IBM_i\n- MySQL\n- Oracle\n- PosgtreSQL\n- SQLServer
Source DatabaseVariableThe name of the database containing the source table.
Source SchemaVariableThe name of the schema containing the source table.
Source TableVariableThe name of the source table.
Cloud Storage URLVariableThe URL of the location where the CDC change event files are stored. This should have the format:\n\ns3:////\nor\nazure://.blob.core.windows.net///\nor\ngs:////
WarehouseVariableThe Snowflake warehouse used to process the data.
Target DatabaseVariableThe Snowflake database containing the external table with change events and the target table.
Target SchemaVariableThe Snowflake schema containing the external table with change events and the target table.
External TableVariableThe external table in Snowflake containing the CDC change events.
Target Table PrefixVariableA prefix to prepend to the source table name to generate the target table name. If no Target Table Prefix is specified, the target table will have the same name as the source table.
Fully Qualify Target Table (Recommended)VariableOptionally includes the source database and schema in the target table name.\n\nY: The target table name will have the format \n__\n\nN: The target table name will have the format \n\n\nIt is strongly recommended to leave this set to 'Y', unless you are confident that your source table names will always be unique.\n\nIf the target table name is not fully qualified, and there are source tables with the same name in different databases or schemas, this job will see them as the same table and attempt to merge their metadata and contents.
Transformation TypeVariableThe type of transformation used when applying the change events to the target table.\n\nCopy Table: The target table will be maintained as a copy of the source table\nCopy Table with Soft Deletes: Same as Copy Table, but records deleted in the source table will be retained in the target table.\nChange Log: All change events will be extracted and appended to the target table\n\nNote: A primary key is required on the source table for Copy Table and Copy Table with Soft Deletes transformations. The primary key is used by the shared job to merge updates into the target table. If the source table does not have a primary key, the transformation type will be updated to Change Log.\n\nWarning: If you intend to change the type of transformation for an already existing / running workflow, you must also drop the target table(s) before changing the transformation type. The transformation types are mutually exclusive and cannot be applied to the same target table(s).
Append All MetadataVariableAppend all metadata columns to the target table. Options are:\n\nY: Append all metadata columns\nN: Only append metadata columns required by this job
Azure Key Vault URLVariableRequired for tagging files in Azure Blob Storage\nThe URL of the Azure Key Vault used to store the Azure Blob Storage connection string. This is also called the vault's \"DNS Name\".
Azure Blob Storage Account Key SecretVariableRequired for tagging files in Azure Blob Storage\nName of the Azure Key Vault secret which stores the Azure Blob Storage account key.
Schema Drift ActionVariableThe action taken when schema changes are detected in the source table, which require an update to the target table:\n\nUpdate Target: The target table is updated according to the new schema.\nFail Job: This job will fail, causing no changes to the target table.
Logging EnabledVariableEnable logging of row counts from transformation jobs. Options are:\n\nY: Log row counts\nN: Do not log row counts\n\nNote: Logged row counts may not match the number of rows processed by the CDC agent. For the Copy Table and Copy Table with Soft Deletes transformation types, if a given record has more than one change event in a job execution, only the final change event will be processed. The \"after\" object of the final change event contains the net changes from all the previous events.
Log TableVariableThe name of the table in Snowflake containing the logged events.
Trim All String ColumnsVariableRemove trailing whitespace from string columns. Options are:\n\nY: Remove trailing whitespace\nN: Do not remove trailing whitespace
Tag Processed FilesVariableOnce the Avro files have been processed, update the tags or metadata in cloud storage. This enables a lifecycle management policy to be applied to files to reduce storage costs. Options are:\n\nY: Update tags / metadata\nN: Do not update tags / metadata\n\nS3 and Azure Blob Storage\nA tag of matillion_cdc_processed = true will be added to each file. If a file already has the maximum 10 tags, the tagging process will be skipped.\n\nNote: For Azure Blob Storage, Blob Index Tags are not supported on Storage Accounts with a hierarchical namespace enabled.\n\nGoogle Cloud Storage\nThe 'Custom-Time' metadata will be updated to be the time the file was processed. If the 'Custom-Time' has already been set to a value in the future, the metadata update will be skipped.\n
Bytes to Decimal FunctionVariableFor some source databases (e.g. PostgreSQL and Oracle), numeric columns can be defined without specifying a scale or precision, and these columns are able to contain values with different scales and precisions. When these columns are extracted to the Avro files, they are denoted as the type VariableScaleDecimal.\n\nA VariableScaleDecimal value is a variant object with two properties, the value stored as a byte array, and a numeric scale. Snowflake SQL cannot natively convert these byte arrays back to the original numeric format. If your source tables contain such columns, a User Defined Function is required by the shared job to convert the values.\n\nIf no value is specified for this parameter, the shared job will not attempt to convert the byte arrays, and will load them to columns of type variant.\n\nIf a value is specified, the shared job will check whether the user defined function already exists. If it does not exist, the job will attempt to create it. If the job cannot create the function due to insufficient privileges, the job will fail and the function must be created in the Snowflake Console.\n\nIf the parameter is set to a value like BYTES_TO_DECIMAL, the job will look for the function in the environment default database and schema.\n\nIf the parameter is set to a value like CDC_DB.PUBLIC.BYTES_TO_DECIMAL, the job will look for the function in the PUBLIC schema of the CDC_DB database.
Use Source SchemasVariableOptionally create the target tables in a schema with the same name as the source table.\nIf the schema doesn't already exist, the job will try to create it.\n\nY: Create the target tables in a schema with the same name as the source.\nN: Create the target tables in the schema specified by the Target Schema parameter.\n\nNote: Which ever option is selected, the external tables will still be created in the schema specified by the Target Schema parameter.
Primary Key OverrideGridOptionally provide a list of primary key columns for the source tables.\n\nBy default, the job will read the primary key columns from the CDC Avro files. However, if the source table does not a have a primary defined in its DDL, a list of unique columns can be specified here to enable Copy Table transformations.\n\nNote: The values for the source_database, source_schema, source_table, and source_column are case sensitive, and must match the source database.
","created":1728919296380},"orchestrationJobs":{"Logging - 1 - Log Metrics":{"id":-2,"revision":1,"created":1728917529394,"timestamp":1728917529394,"components":{"33713":{"id":33713,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":80,"y":80,"width":32,"height":32,"inputConnectorIDs":[38464],"outputSuccessConnectorIDs":[33727],"outputFailureConnectorIDs":[33734],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Build Grid Variables"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"from pprint import pprint\n\nlogging_message = context.getGridVariable('logging_message')\nprint('logging_message:')\npprint(logging_message)\nprint()\n\nif logging_message:\n \n logging_metrics = [\n [f\"{log_level.upper()}_[{context}]\", 'text', values] for context, values in logging_message\n ]\n print('logging_metrics:')\n pprint(logging_metrics)\n context.updateGridVariable('logging_metrics', logging_metrics)\n print()\n\n fixed_flow_data = [\n [\n exported_from_name, \n run_history_id,\n values,\n log_namespace,\n f\"{log_level.upper()}_[{context}]\",\n '',\n ] for context, values in logging_message\n ]\n print('fixed_flow_data:')\n pprint(fixed_flow_data, width=300)\n context.updateGridVariable('fixed_flow_data', fixed_flow_data)\n\n \n "}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"33714":{"id":33714,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-2112089365,"x":240,"y":80,"width":32,"height":32,"inputConnectorIDs":[33727],"outputSuccessConnectorIDs":[33728],"outputFailureConnectorIDs":[33733],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log to Internal Metrics Store"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Metrics","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"logging_metrics"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"key"},"2":{"slot":2,"type":"GRID","value":"type"},"3":{"slot":3,"type":"GRID","value":"value"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${log_namespace}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"33715":{"id":33715,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":-400,"y":80,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[33732],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"33716":{"id":33716,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":1611478312,"x":400,"y":80,"width":32,"height":32,"inputConnectorIDs":[33728],"outputSuccessConnectorIDs":[43036],"outputFailureConnectorIDs":[33735],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Create Log Table in Snowflake"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${database}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${schema}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"New Table Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${log_table}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"table_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"column_name"},"2":{"slot":2,"type":"GRID","value":"data_type"},"3":{"slot":3,"type":"GRID","value":"size"},"4":{"slot":4,"type":"GRID","value":"precision"},"5":{"slot":5,"type":"GRID","value":"default_value"},"6":{"slot":6,"type":"GRID","value":"not_null"},"7":{"slot":7,"type":"GRID","value":"unique"},"8":{"slot":8,"type":"GRID","value":"comment"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Create/Replace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Create if not exists"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Clustering Keys","elements":{},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Data Retention Time in Days","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":""}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Comment","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"11":{"slot":11,"name":"Primary Keys","elements":{},"visible":true,"status":{},"columnNames":[]},"12":{"slot":12,"name":"Table Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Permanent"}}}},"visible":true,"status":{},"columnNames":[]},"13":{"slot":13,"name":"Default DDL Collation","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"33717":{"id":33717,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":-240,"y":80,"width":32,"height":32,"inputConnectorIDs":[33732],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[38433],"outputFalseConnectorIDs":[33731],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Logging Enabled?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Advanced"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"logging_enabled"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"Y"}}}},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":false,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"['yes', 'y', 'true', 't', '1'].indexOf(logging_enabled.toLowerCase()) !== -1"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"33718":{"id":33718,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-193159521,"x":560,"y":80,"width":32,"height":32,"inputConnectorIDs":[43036],"outputSuccessConnectorIDs":[43039],"outputFailureConnectorIDs":[43042],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log to Snowflake Table"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Logging - 1-1 - Append Metrics"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"database"},"2":{"slot":2,"type":"STRING","value":"${database}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"schema"},"2":{"slot":2,"type":"STRING","value":"${schema}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"warehouse"},"2":{"slot":2,"type":"STRING","value":"${warehouse}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"log_table"},"2":{"slot":2,"type":"STRING","value":"${log_table}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"fixed_flow_data"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"componet_name"},"2":{"slot":2,"type":"GRID","value":"task_batch_id"},"3":{"slot":3,"type":"GRID","value":"value"},"4":{"slot":4,"type":"GRID","value":"namespace"},"5":{"slot":5,"type":"GRID","value":"key"},"6":{"slot":6,"type":"GRID","value":"timestamp"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"fixed_flow_data"},"2":{"slot":2,"type":"STRING","value":"grid"}}},"2":{"slot":2,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"table_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"column_name"},"2":{"slot":2,"type":"GRID","value":"data_type"},"3":{"slot":3,"type":"GRID","value":"size"},"4":{"slot":4,"type":"GRID","value":"precision"},"5":{"slot":5,"type":"GRID","value":"default_value"},"6":{"slot":6,"type":"GRID","value":"not_null"},"7":{"slot":7,"type":"GRID","value":"unique"},"8":{"slot":8,"type":"GRID","value":"comment"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"table_metadata"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync Single Table (Exchange)"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Data Loader.CDC"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"3"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"33719":{"id":33719,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":720,"y":80,"width":32,"height":32,"inputConnectorIDs":[43039],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Success - Logging Complete"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"33720":{"id":33720,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":560,"y":240,"width":32,"height":32,"inputConnectorIDs":[43042],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Failure - Logging to Snowflake Table"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"33721":{"id":33721,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":400,"y":240,"width":32,"height":32,"inputConnectorIDs":[33735],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Failure - Create Log Table in Snowflake"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"33722":{"id":33722,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":240,"y":240,"width":32,"height":32,"inputConnectorIDs":[33733],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Failure - Logging to Internal Metrics store"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"33723":{"id":33723,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":80,"y":240,"width":32,"height":32,"inputConnectorIDs":[33734],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Failure - Build Grid Variables"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"33725":{"id":33725,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":-240,"y":240,"width":32,"height":32,"inputConnectorIDs":[33731],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Logging not Enabled - No further processing"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"38432":{"id":38432,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":-80,"y":80,"width":32,"height":32,"inputConnectorIDs":[38433],"outputSuccessConnectorIDs":[38464],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Check log table name"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"log_table = log_table.strip().upper()\n\nprint(f'log_table: {log_table}')\ncontext.updateVariable('log_table', log_table)"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"33727":{"id":33727,"sourceID":33713,"targetID":33714},"33728":{"id":33728,"sourceID":33714,"targetID":33716},"38464":{"id":38464,"sourceID":38432,"targetID":33713},"43036":{"id":43036,"sourceID":33716,"targetID":33718},"43039":{"id":43039,"sourceID":33718,"targetID":33719}},"failureConnectors":{"33733":{"id":33733,"sourceID":33714,"targetID":33722},"33734":{"id":33734,"sourceID":33713,"targetID":33723},"33735":{"id":33735,"sourceID":33716,"targetID":33721},"43042":{"id":43042,"sourceID":33718,"targetID":33720}},"unconditionalConnectors":{"33732":{"id":33732,"sourceID":33715,"targetID":33717}},"trueConnectors":{"38433":{"id":38433,"sourceID":33717,"targetID":38432}},"falseConnectors":{"33731":{"id":33731,"sourceID":33717,"targetID":33725}},"iterationConnectors":{},"noteConnectors":{},"canUndo":false,"undoCommand":"","undoCreated":0,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"33712":{"id":33712,"x":-315,"y":-234,"width":1112,"height":230,"text":"This process utilises the postgres internal metrics store to log messages. This aligns to how MDL Batch stores metrics and metadata about pipeline runs. A job variable, **__logging_enabled__**, is used to toggle logging. \n\nThe target platform metrics table is also created if it doesn't exist and is named as per ${log_table}. \n\nThe calling process should pass the following, which is the same structure as the Logger component in Data Loader - this has been done for consistency across the products:\n* **component_name**- name of the calling component or where the messages are being exported from, to aid traceability.\n* **log_namespace **- identifier for the process being reported - could be a tablename, filename, etc\n* **log_level **- Error, Warning or Info.\n* **schema, database, warehouse**- The target snowflake platform details. \n* **logging_message** (grid) - Context is the type of message, Values is the detail or metric therein. For example, the context might be 'Records loaded' and the value might be '100'.","colour":"e6e63c"}},"variables":{"schema":{"definition":{"name":"schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":null},"database":{"definition":{"name":"database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":null},"log_level":{"definition":{"name":"log_level","type":"TEXT","scope":"BRANCH","description":"Logging level to be used.","visibility":"PUBLIC"},"value":""},"log_table":{"definition":{"name":"log_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":null},"warehouse":{"definition":{"name":"warehouse","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"log_namespace":{"definition":{"name":"log_namespace","type":"TEXT","scope":"BRANCH","description":"Namespace to store all of the metrics inside.","visibility":"PUBLIC"},"value":""},"logging_enabled":{"definition":{"name":"logging_enabled","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":null},"exported_from_name":{"definition":{"name":"exported_from_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""}},"grids":{"table_metadata":{"definition":{"name":"table_metadata","scope":"BRANCH","definitions":[{"name":"column_name","type":"TEXT"},{"name":"data_type","type":"TEXT"},{"name":"size","type":"TEXT"},{"name":"precision","type":"TEXT"},{"name":"default_value","type":"TEXT"},{"name":"not_null","type":"TEXT"},{"name":"unique","type":"TEXT"},{"name":"comment","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[{"values":["TASK_BATCH_ID","NUMBER","38","0","","No","No",""]},{"values":["TIMESTAMP","TIMESTAMP","26","9","","No","No",""]},{"values":["COMPONENT_NAME","VARCHAR","16777216","","","No","No",""]},{"values":["NAMESPACE","VARCHAR","16777216","","","No","No",""]},{"values":["KEY","VARCHAR","16777216","","","No","No",""]},{"values":["VALUE","VARCHAR","16777216","","","No","No",""]}]},"fixed_flow_data":{"definition":{"name":"fixed_flow_data","scope":"TASKBATCH","definitions":[{"name":"componet_name","type":"TEXT"},{"name":"task_batch_id","type":"TEXT"},{"name":"value","type":"TEXT"},{"name":"namespace","type":"TEXT"},{"name":"key","type":"TEXT"},{"name":"timestamp","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[]},"logging_message":{"definition":{"name":"logging_message","scope":"BRANCH","definitions":[{"name":"message_context","type":"TEXT"},{"name":"message_values","type":"TEXT"}],"description":"Messages are key value pairs that map some context for the logging to some values.\nFor example, the MessageContext may be 'Missing Columns', then the MessageValues would be 'column_1, column_2, column_3'.","visibility":"PUBLIC"},"values":[]},"logging_metrics":{"definition":{"name":"logging_metrics","scope":"BRANCH","definitions":[{"name":"key","type":"TEXT"},{"name":"type","type":"TEXT"},{"name":"value","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[]}}},"Sync Single Table - 1-1 - Validate schema":{"id":-2,"revision":1,"created":1728917529394,"timestamp":1728917529394,"components":{"43516":{"id":43516,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":-160,"y":-80,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[43794],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Start","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"43563":{"id":43563,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":160,"y":-80,"width":32,"height":32,"inputConnectorIDs":[43775],"outputSuccessConnectorIDs":[43681],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Set schemas"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"stage_schema = target_schema\n\nif use_source_schemas == 'y':\n target_schema = source_schema.upper()\n \nprint(f'stage_schema: {stage_schema!r}')\ncontext.updateVariable('stage_schema', stage_schema)\n\nprint()\nprint(f'target_schema: {target_schema!r}')\ncontext.updateVariable('target_schema', target_schema)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"43680":{"id":43680,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":400,"y":-80,"width":32,"height":32,"inputConnectorIDs":[43681],"outputSuccessConnectorIDs":[43691],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Build check_schema_sql"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"check_schema_sql = \\\nf\"\"\"SELECT \n count(1) AS \"SCHEMA_EXISTS\"\nFROM \n \"{target_database}\".\"INFORMATION_SCHEMA\".\"SCHEMATA\"\nWHERE \n \"CATALOG_NAME\" = '{target_database}'\n AND \"SCHEMA_NAME\" = '{target_schema}'\"\"\"\n\nprint(f'check_schema_sql: \\n{check_schema_sql}')\ncontext.updateVariable('check_schema_sql', check_schema_sql)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"43685":{"id":43685,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":880,"y":-160,"width":32,"height":32,"inputConnectorIDs":[43727],"outputSuccessConnectorIDs":[43688],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Build create_schema_sql"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"create_schema_sql = f'CREATE SCHEMA \"{target_database}\".\"{target_schema}\"'\n\nprint(f'create_schema_sql: \\n{create_schema_sql}')\ncontext.updateVariable('create_schema_sql', create_schema_sql)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"43687":{"id":43687,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-798585337,"x":1040,"y":-160,"width":32,"height":32,"inputConnectorIDs":[43688],"outputSuccessConnectorIDs":[43781],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Create target schema"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"SQL Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${create_schema_sql}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"43690":{"id":43690,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1032749985,"x":560,"y":-80,"width":32,"height":32,"inputConnectorIDs":[43691],"outputSuccessConnectorIDs":[43696],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Check target schema exists"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"SQL Query","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${check_schema_sql}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Scalar Variable Mapping","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"schema_exists"},"2":{"slot":2,"type":"STRING","value":"SCHEMA_EXISTS"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"[Environment Default]"}}}},"visible":false,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Table Columns","elements":{},"visible":false,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"[Environment Default]"}}}},"visible":false,"status":{},"columnNames":[]},"10":{"slot":10,"name":"Limit","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"100"}}}},"visible":false,"status":{},"columnNames":[]},"11":{"slot":11,"name":"Order By","elements":{},"visible":false,"status":{},"columnNames":[]},"12":{"slot":12,"name":"Sort","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Ascending"}}}},"visible":false,"status":{},"columnNames":[]},"13":{"slot":13,"name":"Filter Conditions","elements":{},"visible":false,"status":{},"columnNames":[]},"14":{"slot":14,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"AND"}}}},"visible":false,"status":{},"columnNames":[]},"20":{"slot":20,"name":"Basic / Advanced","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Advanced"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"43695":{"id":43695,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":720,"y":-80,"width":32,"height":32,"inputConnectorIDs":[43696],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[43731],"outputFalseConnectorIDs":[43727],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Target schema exists?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"schema_exists"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"43700":{"id":43700,"inputCardinality":"MANY","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1343684451,"x":1200,"y":-80,"width":32,"height":32,"inputConnectorIDs":[43731,43781,43838],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[43703],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Join paths"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"43702":{"id":43702,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":1360,"y":-80,"width":32,"height":32,"inputConnectorIDs":[43703],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Success"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"43793":{"id":43793,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":0,"y":-80,"width":32,"height":32,"inputConnectorIDs":[43794],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[43775],"outputFalseConnectorIDs":[43834],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Use source schemas?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"use_source_schemas"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"y"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"43816":{"id":43816,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":160,"y":80,"width":32,"height":32,"inputConnectorIDs":[43834],"outputSuccessConnectorIDs":[43838],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Set stage schema"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"stage_schema = target_schema\n\nprint(f'stage_schema: {stage_schema!r}')\ncontext.updateVariable('stage_schema', stage_schema)\n\nprint()\nprint(f'target_schema: {target_schema!r}')\ncontext.updateVariable('target_schema', target_schema)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"43681":{"id":43681,"sourceID":43563,"targetID":43680},"43688":{"id":43688,"sourceID":43685,"targetID":43687},"43691":{"id":43691,"sourceID":43680,"targetID":43690},"43696":{"id":43696,"sourceID":43690,"targetID":43695},"43781":{"id":43781,"sourceID":43687,"targetID":43700},"43838":{"id":43838,"sourceID":43816,"targetID":43700}},"failureConnectors":{},"unconditionalConnectors":{"43703":{"id":43703,"sourceID":43700,"targetID":43702},"43794":{"id":43794,"sourceID":43516,"targetID":43793}},"trueConnectors":{"43731":{"id":43731,"sourceID":43695,"targetID":43700},"43775":{"id":43775,"sourceID":43793,"targetID":43563}},"falseConnectors":{"43727":{"id":43727,"sourceID":43695,"targetID":43685},"43834":{"id":43834,"sourceID":43793,"targetID":43816}},"iterationConnectors":{},"noteConnectors":{},"canUndo":false,"undoCommand":"","undoCreated":0,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"43941":{"id":43941,"x":-75,"y":-314,"width":308,"height":467,"text":"**Set schema names**\n\nIf use_source_schemas = y then set\n\n- stage_schema = target_schema\n- target_schema = source_schema.upper()\n\nelse\n\n- stage_schema = target_schema\n- target_schema = target_schema","colour":"e6e63c"},"43945":{"id":43945,"x":325,"y":-317,"width":950,"height":315,"text":"**Check target schema exists**\n\nIf using source schemas, check target schema exists.\n\nCreate if not.","colour":"e6e63c"}},"variables":{"stage_schema":{"definition":{"name":"stage_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"schema_exists":{"definition":{"name":"schema_exists","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"source_schema":{"definition":{"name":"source_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_schema":{"definition":{"name":"target_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"TARGET"},"target_database":{"definition":{"name":"target_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"SOL_ENG_TARGET_DB"},"check_schema_sql":{"definition":{"name":"check_schema_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"create_schema_sql":{"definition":{"name":"create_schema_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"use_source_schemas":{"definition":{"name":"use_source_schemas","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""}},"grids":{}},"Sync Single Table - 1-2-1 - Get Avro schema":{"id":-2,"revision":1,"created":1728917529394,"timestamp":1728917529394,"components":{"41486":{"id":41486,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":-800,"y":0,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[41495],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Start","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41487":{"id":41487,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":-160,"y":0,"width":32,"height":32,"inputConnectorIDs":[41494],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Success"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41488":{"id":41488,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":-640,"y":0,"width":32,"height":32,"inputConnectorIDs":[41495],"outputSuccessConnectorIDs":[41493],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Build get_schema_sql"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"get_schema_sql = \\\nf'''SELECT \n coalesce(any_value(\"AVRO_SCHEMA\"), '') AS \"AVRO_SCHEMA\"\nFROM (\n SELECT \n \"VALUE\" AS \"AVRO_SCHEMA\"\n FROM\n \"{target_database}\".\"{stage_schema}\".\"{stage_table}_VERSION_METADATA\"\n WHERE \n \"DATABASE\" = '{source_database}' \n AND \"SCHEMA\" = '{source_schema}' \n AND \"TABLE\" = '{source_table}'\n AND \"VERSION\" = {source_version}\n)'''\n\nprint(f'get_schema_sql: \\n{get_schema_sql}')\ncontext.updateVariable('get_schema_sql', get_schema_sql)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41489":{"id":41489,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1032749985,"x":-480,"y":0,"width":32,"height":32,"inputConnectorIDs":[41493],"outputSuccessConnectorIDs":[41492],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Read Avro schema from external table"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"SQL Query","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${get_schema_sql}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Scalar Variable Mapping","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"avro_schema"},"2":{"slot":2,"type":"STRING","value":"AVRO_SCHEMA"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"[Environment Default]"}}}},"visible":false,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Table Columns","elements":{},"visible":false,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"[Environment Default]"}}}},"visible":false,"status":{},"columnNames":[]},"10":{"slot":10,"name":"Limit","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"100"}}}},"visible":false,"status":{},"columnNames":[]},"11":{"slot":11,"name":"Order By","elements":{},"visible":false,"status":{},"columnNames":[]},"12":{"slot":12,"name":"Sort","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Ascending"}}}},"visible":false,"status":{},"columnNames":[]},"13":{"slot":13,"name":"Filter Conditions","elements":{},"visible":false,"status":{},"columnNames":[]},"14":{"slot":14,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"AND"}}}},"visible":false,"status":{},"columnNames":[]},"20":{"slot":20,"name":"Basic / Advanced","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Advanced"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41490":{"id":41490,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":-320,"y":0,"width":32,"height":32,"inputConnectorIDs":[41492],"outputSuccessConnectorIDs":[41494],"outputFailureConnectorIDs":[41496],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Check Avro schema is valid JSON"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"print(f'avro_schema: {avro_schema!r}')\n\n# check that schema is valid JSON\ntry:\n json_schema = json.loads(avro_schema)\n print('avro_schema is valid JSON')\nexcept Exception as e:\n err_msg = f'There may be an issue with the JSON representation of the Avro schema.\\nException: {e!r}'\n raise TypeError(err_msg)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41491":{"id":41491,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":-317,"y":162,"width":32,"height":32,"inputConnectorIDs":[41496],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Failure - Avro schema is not valid JSON"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"41492":{"id":41492,"sourceID":41489,"targetID":41490},"41493":{"id":41493,"sourceID":41488,"targetID":41489},"41494":{"id":41494,"sourceID":41490,"targetID":41487}},"failureConnectors":{"41496":{"id":41496,"sourceID":41490,"targetID":41491}},"unconditionalConnectors":{"41495":{"id":41495,"sourceID":41486,"targetID":41488}},"trueConnectors":{},"falseConnectors":{},"iterationConnectors":{},"noteConnectors":{},"canUndo":false,"undoCommand":"","undoCreated":0,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"41485":{"id":41485,"x":-714,"y":-154,"width":469,"height":229,"text":"**Get Avro schema**\n\nRead the Avro schema from the external table \"_VERSION_METADATA\"","colour":"e6e63c"}},"variables":{"avro_schema":{"definition":{"name":"avro_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"stage_table":{"definition":{"name":"stage_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"source_table":{"definition":{"name":"source_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":null},"stage_schema":{"definition":{"name":"stage_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":null},"source_schema":{"definition":{"name":"source_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":null},"get_schema_sql":{"definition":{"name":"get_schema_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"source_version":{"definition":{"name":"source_version","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":null},"source_database":{"definition":{"name":"source_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":null},"target_database":{"definition":{"name":"target_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":null}},"grids":{}},"Sync Single Table - 1-2-2 - Validate event metadata":{"id":-2,"revision":1,"created":1728917529394,"timestamp":1728917529394,"components":{"44304":{"id":44304,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":0,"y":0,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[44559],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Start","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44335":{"id":44335,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":320,"y":0,"width":32,"height":32,"inputConnectorIDs":[44563],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[44339],"outputFalseConnectorIDs":[44566],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Connector is postgresql?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"connector"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"postgresql"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44336":{"id":44336,"inputCardinality":"MANY","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1343684451,"x":640,"y":0,"width":32,"height":32,"inputConnectorIDs":[44400,44338,44566],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[44437],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Join paths"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44337":{"id":44337,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":480,"y":80,"width":32,"height":32,"inputConnectorIDs":[44339],"outputSuccessConnectorIDs":[44338],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Check for sequence"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"from decimal import Decimal\n\n\nevent_metadata = context.getGridVariable('event_metadata') or []\nrequired_cols = ['sequence']\n\nfor column, *_ in event_metadata:\n if column in required_cols:\n required_cols.remove(column)\n\nif not required_cols:\n print('All required columns found.')\n exit()\n \n# if sequence is missing, fall back to using lsn\nprint(\n f'Unable to find the following fields in the event metadata: {required_cols!r}\\n'\n \"Falling back to using 'lsn'.\"\n)\nprint()\n\ncid = 'lsn'\ncid_dt = 'NUMBER'\ncid_p = Decimal(38)\ncid_s = Decimal(0)\ncid_dbt = 'NUMBER(38, 0)'\n\nprint(f'commit_id_name: {cid!r}')\ncontext.updateVariable('commit_id_name', cid)\n\nprint(f'commit_id_data_type: {cid_dt!r}')\ncontext.updateVariable('commit_id_datatype', cid_dt)\n\nprint(f'commit_id_precision: {cid_p!r}')\ncontext.updateVariable('commit_id_precision', cid_p)\n\nprint(f'commit_id_scale: {cid_s!r}')\ncontext.updateVariable('commit_id_scale', cid_s)\n\nprint(f'commit_id_database_type: {cid_dbt!r}')\ncontext.updateVariable('commit_id_database_type', cid_dbt)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44371":{"id":44371,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":160,"y":0,"width":32,"height":32,"inputConnectorIDs":[44559],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[44391],"outputFalseConnectorIDs":[44563],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Connector is oracle?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"connector"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"oracle"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44390":{"id":44390,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":480,"y":160,"width":32,"height":32,"inputConnectorIDs":[44391],"outputSuccessConnectorIDs":[44400],"outputFailureConnectorIDs":[44570],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Check for rs_id and ssn"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"event_metadata = context.getGridVariable('event_metadata') or []\nrequired_cols = ['rs_id', 'ssn']\n\nfor column, *_ in event_metadata:\n if column in required_cols:\n required_cols.remove(column)\n\nif not required_cols:\n print('All required columns found.')\n exit()\n\n# if either rs_id or ssn are missing, raise error\nerr_msg = \\\nf'''For Oracle pipelines, this revision of the shared job requires the following fields\nwhich could not be found in the change event metadata: {required_cols!r}.\nYou must snapshot the source tables with a minimum agent version of 2.83.4.'''\n\nraise Exception(err_msg)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44436":{"id":44436,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":800,"y":0,"width":32,"height":32,"inputConnectorIDs":[44437],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Success"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44454":{"id":44454,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":478,"y":321,"width":32,"height":32,"inputConnectorIDs":[44570],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Failure - Files produced by older agent version"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"44338":{"id":44338,"sourceID":44337,"targetID":44336},"44400":{"id":44400,"sourceID":44390,"targetID":44336}},"failureConnectors":{"44570":{"id":44570,"sourceID":44390,"targetID":44454}},"unconditionalConnectors":{"44437":{"id":44437,"sourceID":44336,"targetID":44436},"44559":{"id":44559,"sourceID":44304,"targetID":44371}},"trueConnectors":{"44339":{"id":44339,"sourceID":44335,"targetID":44337},"44391":{"id":44391,"sourceID":44371,"targetID":44390}},"falseConnectors":{"44563":{"id":44563,"sourceID":44371,"targetID":44335},"44566":{"id":44566,"sourceID":44335,"targetID":44336}},"iterationConnectors":{},"noteConnectors":{},"canUndo":false,"undoCommand":"","undoCreated":0,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"44341":{"id":44341,"x":85,"y":-234,"width":628,"height":474,"text":"**Check event metadata**\n\nWith rev 3, we are using\n\n* __metadata.sequence__to sort the change events for Postgres sources. If the current files do not contain this field, fall back to __metadata.lsn__.\n\n* __rs_id__ and __ssn__to sort change events for Oracle sources. If either field is missing, fail the job execution\n","colour":"e6e63c"}},"variables":{"connector":{"definition":{"name":"connector","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"commit_id_name":{"definition":{"name":"commit_id_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"commit_id_scale":{"definition":{"name":"commit_id_scale","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"commit_id_datatype":{"definition":{"name":"commit_id_datatype","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"commit_id_precision":{"definition":{"name":"commit_id_precision","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"commit_id_database_type":{"definition":{"name":"commit_id_database_type","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""}},"grids":{"event_metadata":{"definition":{"name":"event_metadata","scope":"TASKBATCH","definitions":[{"name":"property","type":"TEXT"},{"name":"format_string","type":"TEXT"},{"name":"type","type":"TEXT"},{"name":"size","type":"DECIMAL"},{"name":"precision","type":"DECIMAL"},{"name":"database_type","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]}}},"Sync Single Table - 1-2-3 - Build new target table metadata":{"id":-2,"revision":1,"created":1728917529394,"timestamp":1728917529394,"components":{"41384":{"id":41384,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":880,"y":240,"width":32,"height":32,"inputConnectorIDs":[41409],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[41403],"outputFalseConnectorIDs":[41407],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Apply soft deletes?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"transformation_type"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"copy table with soft deletes"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Or"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"['yes', 'y', 'true', 't', '1'].indexOf(soft_deletes.toLowerCase()) !== -1"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41385":{"id":41385,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-369209785,"x":1040,"y":160,"width":32,"height":32,"inputConnectorIDs":[41403],"outputSuccessConnectorIDs":[41397],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Append Deleted column to target_metadata"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Target Grid Variable","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"target_metadata"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Fixed/Grid","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Fixed"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Values Grid","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_DELETED"},"2":{"slot":2,"type":"STRING","value":"BOOLEAN"},"3":{"slot":3,"type":"STRING","value":"0"},"4":{"slot":4,"type":"STRING","value":"0"},"5":{"slot":5,"type":"STRING","value":"BOOLEAN"},"6":{"slot":6,"type":"STRING","value":null},"7":{"slot":7,"type":"STRING","value":"No"},"8":{"slot":8,"type":"STRING","value":"No"},"9":{"slot":9,"type":"STRING","value":"No"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Source Grid Variable","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Column Mapping","elements":{},"visible":false,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Append/Prepend","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Append"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41386":{"id":41386,"inputCardinality":"MANY","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1343684451,"x":1200,"y":240,"width":32,"height":32,"inputConnectorIDs":[41397,41407],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[45847],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Join paths (2)"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41387":{"id":41387,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-369209785,"x":80,"y":240,"width":32,"height":32,"inputConnectorIDs":[41411],"outputSuccessConnectorIDs":[41401],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Create target_metadata from nested_metadata"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Target Grid Variable","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"target_metadata"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Fixed/Grid","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Grid"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Values Grid","elements":{},"visible":false,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Source Grid Variable","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"nested_metadata"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Column Mapping","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"alias"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"type"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"size"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"precision"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"database_type"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"default_value"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"not_null"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"unique"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Append/Prepend","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Append"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41388":{"id":41388,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":240,"y":240,"width":32,"height":32,"inputConnectorIDs":[41401],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[41405],"outputFalseConnectorIDs":[41408],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Append full CDC metadata to target_metadata?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Advanced"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"append_metadata"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"Y"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"append_metadata"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"y"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"append_metadata"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"Yes"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"append_metadata"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"YES"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"append_metadata"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"yes"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"append_metadata"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"True"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"append_metadata"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"true"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"append_metadata"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"TRUE"}}}},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Or"}}}},"visible":false,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"['yes', 'y', 'true', 't', '1'].indexOf(append_metadata.toLowerCase()) !== -1"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41389":{"id":41389,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-369209785,"x":560,"y":320,"width":32,"height":32,"inputConnectorIDs":[41406],"outputSuccessConnectorIDs":[41398],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Append minimum CDC metadata for Copy Table"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Target Grid Variable","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"target_metadata"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Fixed/Grid","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Fixed"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Values Grid","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_SRC_VERSION"},"2":{"slot":2,"type":"STRING","value":"NUMBER"},"3":{"slot":3,"type":"DECIMAL","value":"38"},"4":{"slot":4,"type":"DECIMAL","value":"0"},"5":{"slot":5,"type":"STRING","value":"NUMBER(38, 0)"},"6":{"slot":6,"type":"STRING","value":""},"7":{"slot":7,"type":"STRING","value":"No"},"8":{"slot":8,"type":"STRING","value":"No"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_PROCESSED_DATE_HOUR"},"2":{"slot":2,"type":"STRING","value":"TIMESTAMP"},"3":{"slot":3,"type":"DECIMAL","value":"9"},"4":{"slot":4,"type":"DECIMAL","value":"0"},"5":{"slot":5,"type":"STRING","value":"TIMESTAMP"},"6":{"slot":6,"type":"STRING","value":""},"7":{"slot":7,"type":"STRING","value":"No"},"8":{"slot":8,"type":"STRING","value":"No"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_FILENAME"},"2":{"slot":2,"type":"STRING","value":"VARCHAR"},"3":{"slot":3,"type":"DECIMAL","value":"16777216"},"4":{"slot":4,"type":"DECIMAL","value":"0"},"5":{"slot":5,"type":"STRING","value":"VARCHAR(16777216)"},"6":{"slot":6,"type":"STRING","value":""},"7":{"slot":7,"type":"STRING","value":"No"},"8":{"slot":8,"type":"STRING","value":"No"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_SEQUENCE_NUMBER"},"2":{"slot":2,"type":"STRING","value":"${commit_id_datatype}"},"3":{"slot":3,"type":"DECIMAL","value":"${commit_id_precision}"},"4":{"slot":4,"type":"DECIMAL","value":"${commit_id_scale}"},"5":{"slot":5,"type":"STRING","value":"${commit_id_database_type}"},"6":{"slot":6,"type":"STRING","value":""},"7":{"slot":7,"type":"STRING","value":"No"},"8":{"slot":8,"type":"STRING","value":"No"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Source Grid Variable","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Column Mapping","elements":{},"visible":false,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Append/Prepend","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Append"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41390":{"id":41390,"inputCardinality":"MANY","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1343684451,"x":720,"y":240,"width":32,"height":32,"inputConnectorIDs":[41398,41399,41402],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[41409],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Join paths"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41391":{"id":41391,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-369209785,"x":560,"y":160,"width":32,"height":32,"inputConnectorIDs":[41405],"outputSuccessConnectorIDs":[41399],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Append full CDC metadata to target_metadata"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Target Grid Variable","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"target_metadata"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Fixed/Grid","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Fixed"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Values Grid","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_SRC_DATABASE"},"2":{"slot":2,"type":"STRING","value":"VARCHAR"},"3":{"slot":3,"type":"STRING","value":"16777216"},"4":{"slot":4,"type":"STRING","value":"0"},"5":{"slot":5,"type":"STRING","value":"VARCHAR(16777216)"},"6":{"slot":6,"type":"STRING","value":""},"7":{"slot":7,"type":"STRING","value":"No"},"8":{"slot":8,"type":"STRING","value":"No"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_SRC_SCHEMA"},"2":{"slot":2,"type":"STRING","value":"VARCHAR"},"3":{"slot":3,"type":"STRING","value":"16777216"},"4":{"slot":4,"type":"STRING","value":"0"},"5":{"slot":5,"type":"STRING","value":"VARCHAR(16777216)"},"6":{"slot":6,"type":"STRING","value":""},"7":{"slot":7,"type":"STRING","value":"No"},"8":{"slot":8,"type":"STRING","value":"No"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_SRC_TABLE"},"2":{"slot":2,"type":"STRING","value":"VARCHAR"},"3":{"slot":3,"type":"STRING","value":"16777216"},"4":{"slot":4,"type":"STRING","value":"0"},"5":{"slot":5,"type":"STRING","value":"VARCHAR(16777216)"},"6":{"slot":6,"type":"STRING","value":""},"7":{"slot":7,"type":"STRING","value":"No"},"8":{"slot":8,"type":"STRING","value":"No"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_SRC_VERSION"},"2":{"slot":2,"type":"STRING","value":"NUMBER"},"3":{"slot":3,"type":"STRING","value":"38"},"4":{"slot":4,"type":"STRING","value":"0"},"5":{"slot":5,"type":"STRING","value":"NUMBER(38, 0)"},"6":{"slot":6,"type":"STRING","value":""},"7":{"slot":7,"type":"STRING","value":"No"},"8":{"slot":8,"type":"STRING","value":"No"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_LAST_CHANGE_TYPE"},"2":{"slot":2,"type":"STRING","value":"VARCHAR"},"3":{"slot":3,"type":"STRING","value":"16777216"},"4":{"slot":4,"type":"STRING","value":"0"},"5":{"slot":5,"type":"STRING","value":"VARCHAR(16777216)"},"6":{"slot":6,"type":"STRING","value":""},"7":{"slot":7,"type":"STRING","value":"No"},"8":{"slot":8,"type":"STRING","value":"No"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_LAST_COMMIT_TIMESTAMP"},"2":{"slot":2,"type":"STRING","value":"NUMBER"},"3":{"slot":3,"type":"STRING","value":"38"},"4":{"slot":4,"type":"STRING","value":"0"},"5":{"slot":5,"type":"STRING","value":"NUMBER(38, 0)"},"6":{"slot":6,"type":"STRING","value":""},"7":{"slot":7,"type":"STRING","value":"No"},"8":{"slot":8,"type":"STRING","value":"No"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_PROCESSED_DATE_HOUR"},"2":{"slot":2,"type":"STRING","value":"TIMESTAMP"},"3":{"slot":3,"type":"STRING","value":"9"},"4":{"slot":4,"type":"STRING","value":"0"},"5":{"slot":5,"type":"STRING","value":"TIMESTAMP"},"6":{"slot":6,"type":"STRING","value":""},"7":{"slot":7,"type":"STRING","value":"No"},"8":{"slot":8,"type":"STRING","value":"No"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_FILENAME"},"2":{"slot":2,"type":"STRING","value":"VARCHAR"},"3":{"slot":3,"type":"STRING","value":"16777216"},"4":{"slot":4,"type":"STRING","value":"0"},"5":{"slot":5,"type":"STRING","value":"VARCHAR(16777216)"},"6":{"slot":6,"type":"STRING","value":""},"7":{"slot":7,"type":"STRING","value":"No"},"8":{"slot":8,"type":"STRING","value":"No"}}},"9":{"slot":9,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_FILEPATH"},"2":{"slot":2,"type":"STRING","value":"VARCHAR"},"3":{"slot":3,"type":"DECIMAL","value":"16777216"},"4":{"slot":4,"type":"DECIMAL","value":"0"},"5":{"slot":5,"type":"STRING","value":"VARCHAR(16777216)"},"6":{"slot":6,"type":"STRING","value":""},"7":{"slot":7,"type":"STRING","value":"No"},"8":{"slot":8,"type":"STRING","value":"No"}}},"10":{"slot":10,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_SEQUENCE_NUMBER"},"2":{"slot":2,"type":"STRING","value":"${commit_id_datatype}"},"3":{"slot":3,"type":"STRING","value":"${commit_id_precision}"},"4":{"slot":4,"type":"STRING","value":"${commit_id_scale}"},"5":{"slot":5,"type":"STRING","value":"${commit_id_database_type}"},"6":{"slot":6,"type":"STRING","value":""},"7":{"slot":7,"type":"STRING","value":"No"},"8":{"slot":8,"type":"STRING","value":"No"}}},"11":{"slot":11,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_LOAD_BATCH_ID"},"2":{"slot":2,"type":"STRING","value":"NUMBER"},"3":{"slot":3,"type":"STRING","value":"38"},"4":{"slot":4,"type":"STRING","value":"0"},"5":{"slot":5,"type":"STRING","value":"NUMBER(38, 0)"},"6":{"slot":6,"type":"STRING","value":""},"7":{"slot":7,"type":"STRING","value":"No"},"8":{"slot":8,"type":"STRING","value":"No"}}},"12":{"slot":12,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_LOAD_TIMESTAMP"},"2":{"slot":2,"type":"STRING","value":"TIMESTAMP"},"3":{"slot":3,"type":"STRING","value":"9"},"4":{"slot":4,"type":"STRING","value":"0"},"5":{"slot":5,"type":"STRING","value":"TIMESTAMP"},"6":{"slot":6,"type":"STRING","value":""},"7":{"slot":7,"type":"STRING","value":"No"},"8":{"slot":8,"type":"STRING","value":"No"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Source Grid Variable","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Column Mapping","elements":{},"visible":false,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Append/Prepend","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Append"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41393":{"id":41393,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-369209785,"x":560,"y":240,"width":32,"height":32,"inputConnectorIDs":[41404],"outputSuccessConnectorIDs":[41402],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Append minimum CDC metadata for Change Log"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Target Grid Variable","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"target_metadata"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Fixed/Grid","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Fixed"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Values Grid","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_SRC_VERSION"},"2":{"slot":2,"type":"STRING","value":"NUMBER"},"3":{"slot":3,"type":"STRING","value":"38"},"4":{"slot":4,"type":"STRING","value":"0"},"5":{"slot":5,"type":"STRING","value":"NUMBER(38, 0)"},"6":{"slot":6,"type":"STRING","value":""},"7":{"slot":7,"type":"STRING","value":"No"},"8":{"slot":8,"type":"STRING","value":"No"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_LAST_CHANGE_TYPE"},"2":{"slot":2,"type":"STRING","value":"VARCHAR"},"3":{"slot":3,"type":"STRING","value":"16777216"},"4":{"slot":4,"type":"STRING","value":"0"},"5":{"slot":5,"type":"STRING","value":"VARCHAR(16777216)"},"6":{"slot":6,"type":"STRING","value":""},"7":{"slot":7,"type":"STRING","value":"No"},"8":{"slot":8,"type":"STRING","value":"No"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_PROCESSED_DATE_HOUR"},"2":{"slot":2,"type":"STRING","value":"TIMESTAMP"},"3":{"slot":3,"type":"STRING","value":"9"},"4":{"slot":4,"type":"STRING","value":"0"},"5":{"slot":5,"type":"STRING","value":"TIMESTAMP"},"6":{"slot":6,"type":"STRING","value":""},"7":{"slot":7,"type":"STRING","value":"No"},"8":{"slot":8,"type":"STRING","value":"No"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_FILENAME"},"2":{"slot":2,"type":"STRING","value":"VARCHAR"},"3":{"slot":3,"type":"STRING","value":"16777216"},"4":{"slot":4,"type":"STRING","value":"0"},"5":{"slot":5,"type":"STRING","value":"VARCHAR(16777216)"},"6":{"slot":6,"type":"STRING","value":""},"7":{"slot":7,"type":"STRING","value":"No"},"8":{"slot":8,"type":"STRING","value":"No"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_SEQUENCE_NUMBER"},"2":{"slot":2,"type":"STRING","value":"${commit_id_datatype}"},"3":{"slot":3,"type":"STRING","value":"${commit_id_precision}"},"4":{"slot":4,"type":"STRING","value":"${commit_id_scale}"},"5":{"slot":5,"type":"STRING","value":"${commit_id_database_type}"},"6":{"slot":6,"type":"STRING","value":""},"7":{"slot":7,"type":"STRING","value":"No"},"8":{"slot":8,"type":"STRING","value":"No"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Source Grid Variable","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Column Mapping","elements":{},"visible":false,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Append/Prepend","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Append"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41394":{"id":41394,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":400,"y":240,"width":32,"height":32,"inputConnectorIDs":[41408],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[41404],"outputFalseConnectorIDs":[41406],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Transformation is Change Log?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"transformation_type"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"change log"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41395":{"id":41395,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":-80,"y":240,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[41411],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41396":{"id":41396,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":1520,"y":240,"width":32,"height":32,"inputConnectorIDs":[45850],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Success"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"45824":{"id":45824,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":1360,"y":240,"width":32,"height":32,"inputConnectorIDs":[45847],"outputSuccessConnectorIDs":[45850],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Print target_metadata"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"nested_metadata = context.getGridVariable('nested_metadata') or []\nprint('nested_metadata:')\nfor row in nested_metadata:\n print(\"\\t\".join(str(col) for col in row))\n\nprint()\ntarget_metadata = context.getGridVariable('target_metadata') or []\nprint('target_metadata:')\nfor row in target_metadata:\n print(\"\\t\".join(str(col) for col in row))\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"41397":{"id":41397,"sourceID":41385,"targetID":41386},"41398":{"id":41398,"sourceID":41389,"targetID":41390},"41399":{"id":41399,"sourceID":41391,"targetID":41390},"41401":{"id":41401,"sourceID":41387,"targetID":41388},"41402":{"id":41402,"sourceID":41393,"targetID":41390},"45850":{"id":45850,"sourceID":45824,"targetID":41396}},"failureConnectors":{},"unconditionalConnectors":{"41409":{"id":41409,"sourceID":41390,"targetID":41384},"41411":{"id":41411,"sourceID":41395,"targetID":41387},"45847":{"id":45847,"sourceID":41386,"targetID":45824}},"trueConnectors":{"41403":{"id":41403,"sourceID":41384,"targetID":41385},"41404":{"id":41404,"sourceID":41394,"targetID":41393},"41405":{"id":41405,"sourceID":41388,"targetID":41391}},"falseConnectors":{"41406":{"id":41406,"sourceID":41394,"targetID":41389},"41407":{"id":41407,"sourceID":41384,"targetID":41386},"41408":{"id":41408,"sourceID":41388,"targetID":41394}},"iterationConnectors":{},"noteConnectors":{},"canUndo":false,"undoCommand":"","undoCreated":0,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"41383":{"id":41383,"x":6,"y":5,"width":1431,"height":389,"text":"**Build target metadata for current source_version**\n\nUse __nested_metadata__to create __target_metadata__\n\nEither add all CDC metadata columns to __target_metadata__, or just those required by the current __transformation_type__\n\nIf implementing soft deletes, add the MTLN_CDC_DELETED column to the __target_metadata__.","colour":"e6e63c"}},"variables":{"append_metadata":{"definition":{"name":"append_metadata","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"commit_id_scale":{"definition":{"name":"commit_id_scale","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"commit_id_datatype":{"definition":{"name":"commit_id_datatype","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"commit_id_precision":{"definition":{"name":"commit_id_precision","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"transformation_type":{"definition":{"name":"transformation_type","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"commit_id_database_type":{"definition":{"name":"commit_id_database_type","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""}},"grids":{"primary_key":{"definition":{"name":"primary_key","scope":"TASKBATCH","definitions":[{"name":"key_column","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]},"nested_metadata":{"definition":{"name":"nested_metadata","scope":"TASKBATCH","definitions":[{"name":"property","type":"TEXT"},{"name":"alias","type":"TEXT"},{"name":"format_string","type":"TEXT"},{"name":"type","type":"TEXT"},{"name":"size","type":"DECIMAL"},{"name":"precision","type":"DECIMAL"},{"name":"database_type","type":"TEXT"},{"name":"default_value","type":"TEXT"},{"name":"not_null","type":"TEXT"},{"name":"unique","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]},"target_metadata":{"definition":{"name":"target_metadata","scope":"TASKBATCH","definitions":[{"name":"column_name","type":"TEXT"},{"name":"type","type":"TEXT"},{"name":"size","type":"DECIMAL"},{"name":"precision","type":"DECIMAL"},{"name":"database_type","type":"TEXT"},{"name":"default_value","type":"TEXT"},{"name":"not_null","type":"TEXT"},{"name":"unique","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]}}},"Sync Single Table - 1-2 - Process source metadata version":{"id":-2,"revision":1,"created":1728917529394,"timestamp":1728917529394,"components":{"41453":{"id":41453,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":3520,"y":240,"width":32,"height":32,"inputConnectorIDs":[46082],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Success - All new data processed"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41454":{"id":41454,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":2640,"y":400,"width":32,"height":32,"inputConnectorIDs":[41517],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Failure - Error processing new data"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41455":{"id":41455,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":240,"y":400,"width":32,"height":32,"inputConnectorIDs":[41518],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Failure - Unable to parse metadata"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41456":{"id":41456,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":80,"y":240,"width":32,"height":32,"inputConnectorIDs":[41515],"outputSuccessConnectorIDs":[41471],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"avro_schema","mapTo":"avro_schema","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Get Avro schema"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync Single Table - 1-2-1 - Get Avro schema"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"target_database"},"2":{"slot":2,"type":"STRING","value":"${target_database}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"stage_schema"},"2":{"slot":2,"type":"STRING","value":"${stage_schema}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"stage_table"},"2":{"slot":2,"type":"STRING","value":"${stage_table}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"source_database"},"2":{"slot":2,"type":"STRING","value":"${source_database}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"source_schema"},"2":{"slot":2,"type":"STRING","value":"${source_schema}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"source_table"},"2":{"slot":2,"type":"STRING","value":"${source_table}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"source_version"},"2":{"slot":2,"type":"STRING","value":"${source_version}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync Single Table (Exchange)"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Data Loader.CDC"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"3"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41457":{"id":41457,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":240,"y":240,"width":32,"height":32,"inputConnectorIDs":[41471],"outputSuccessConnectorIDs":[44650],"outputFailureConnectorIDs":[41518],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Extract nested_metadata from Avro schema"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"\"\"\"Convert Avro schema to nested_metadata grid variable.\"\"\"\nimport json\nfrom decimal import Decimal\nfrom typing import Any\nfrom typing import Dict\nfrom typing import List\nfrom typing import Union\n\n\nif __package__:\n from . import *\n\nPRIMITIVE_TYPES = ('boolean', 'int', 'long', 'float', 'double', 'bytes', 'string')\n\n\nclass DataType:\n \"\"\"Class to hold a primitive data type and return the details in various string formats.\"\"\"\n\n def __init__(self, name: str) -> None:\n \"\"\"Initialise the class.\"\"\"\n # map from Avro data type to Matillion UI data type, size and precision fields\n self.metl_type_map = {\n 'boolean': 'BOOLEAN',\n 'int': 'NUMBER',\n 'long': 'NUMBER',\n 'float': 'FLOAT',\n 'double': 'FLOAT',\n 'bytes': 'VARIANT',\n 'string': 'VARCHAR',\n }\n self.metl_size_map = {\n 'int': Decimal(38),\n 'long': Decimal(38),\n 'string': Decimal(16777216),\n }\n self.metl_precision_map: Dict[str, Decimal] = {}\n\n # map from Avro data type to Snowflake data type\n self.database_type_map = {\n 'boolean': 'BOOLEAN',\n 'int': 'NUMBER(38,0)',\n 'long': 'NUMBER(38,0)',\n 'float': 'FLOAT',\n 'double': 'FLOAT',\n 'bytes': 'VARIANT',\n 'string': 'VARCHAR(16777216)',\n }\n\n self.name = name\n\n def __repr__(self) -> str:\n \"\"\"String representation of the class.\"\"\"\n return f'DataType(name={self.name})'\n\n def format_string(self) -> str:\n \"\"\"Return format string for column.\"\"\"\n if trim_strings() and self.name == 'string':\n return 'RTRIM({col})'\n else:\n return ''\n\n def metadata_grid(self) -> List[Union[Decimal, str]]:\n \"\"\"Return the data type as a list, used to populate nested_metadata grid variable.\n\n Returns:\n List with format [format_string, metl_type, metl_precision, metl_scale, snowflake_type]\n \"\"\"\n return [\n self.format_string(),\n self.metl_type_map.get(self.name, self.name.upper()),\n self.metl_size_map.get(self.name, Decimal(0)),\n self.metl_precision_map.get(self.name, Decimal(0)),\n self.database_type_map.get(self.name, self.name.upper()),\n ]\n\n\nclass Field:\n \"\"\"Class to hold field name and data type.\"\"\"\n\n def __init__(self, name: str, data_type: DataType) -> None:\n \"\"\"Initialise the class.\"\"\"\n self.name = name\n self.data_type = data_type\n\n def __repr__(self) -> str:\n \"\"\"String representation of the class.\"\"\"\n return f'Field(name={self.name}, data_type={self.data_type})'\n\n\nclass LogicalDataType(DataType):\n \"\"\"Class to hold logical data type and return the details in various string formats.\"\"\"\n\n def __init__(\n self, name: str, logical_type: str, properties: Dict[str, Any]\n ) -> None:\n \"\"\"Initialise the class.\"\"\"\n super().__init__(name=name)\n self.logical_type = logical_type\n self.properties = properties\n self.bytes_to_decimal_function = get_bytes_to_decimal_function()\n\n def __repr__(self) -> str:\n \"\"\"String representation of the class.\"\"\"\n return (\n f'LogicalDataType(name={self.name}, logical_type={self.logical_type}, '\n f'properties={self.properties})'\n )\n\n def metadata_grid(self) -> List[Union[Decimal, str]]:\n \"\"\"Return the data type as a list, used to populate nested_metadata grid variable.\n\n Returns:\n List with format [format_string, metl_type, metl_precision, metl_scale, snowflake_type]\n \"\"\"\n if self.name == 'bytes' and self.logical_type == 'decimal':\n\n precision = self.properties.get('precision', 10)\n scale = self.properties.get('scale', 5)\n\n if precision <= 38:\n return [\n '',\n 'NUMBER',\n Decimal(precision),\n Decimal(scale),\n f'NUMBER({precision},{scale})',\n ]\n elif self.bytes_to_decimal_function:\n return [\n f'{self.bytes_to_decimal_function}({{col}}::VARCHAR, {scale}, {precision})::VARIANT',\n 'VARIANT',\n Decimal(0),\n Decimal(0),\n 'VARIANT',\n ]\n else:\n return [\n '',\n 'VARIANT',\n Decimal(0),\n Decimal(0),\n 'VARIANT',\n ]\n\n elif self.name == 'long' and self.logical_type == 'timestamp-millis':\n return [\n 'TO_TIMESTAMP({col}::NUMBER(38,0), 3)',\n 'TIMESTAMP',\n Decimal(9),\n Decimal(0),\n 'TIMESTAMP',\n ]\n\n elif self.name == 'int' and self.logical_type == 'time-millis':\n return [\n \"TIMEADD(millisecond, {col}::NUMBER(38,0), TIME('00:00:00'))\",\n 'TIME',\n Decimal(3),\n Decimal(0),\n 'TIME(3)',\n ]\n\n elif self.name == 'int' and self.logical_type == 'date':\n return [\n \"DATEADD(day, {col}::NUMBER(38,0), DATE('1970-01-01'))\",\n 'DATE',\n Decimal(0),\n Decimal(0),\n 'DATE',\n ]\n\n else:\n return super().metadata_grid()\n\n\nclass UnknownDataType(DataType):\n \"\"\"Class to hold unknown data type and return the details in various string formats.\"\"\"\n\n def __init__(self, name: str) -> None:\n \"\"\"Initialise the class.\"\"\"\n super().__init__(name=name)\n\n def __repr__(self) -> str:\n \"\"\"String representation of the class.\"\"\"\n return f'UnknownDataType(name={self.name})'\n\n def metadata_grid(self) -> List[Union[Decimal, str]]:\n \"\"\"Return the data type as a list, used to populate nested_metadata grid variable.\n\n Returns:\n List with format [format_string, metl_type, metl_precision, metl_scale, snowflake_type]\n \"\"\"\n return [\n '',\n 'VARIANT',\n Decimal(0),\n Decimal(0),\n 'VARIANT',\n ]\n\n\nclass ArrayDataType(DataType):\n \"\"\"Class to hold array data type and return the details in various string formats.\"\"\"\n\n def __init__(self, element_type: DataType) -> None:\n \"\"\"Initialise the class.\"\"\"\n super().__init__(name='array')\n self.element_type = element_type\n\n def __repr__(self) -> str:\n \"\"\"String representation of the class.\"\"\"\n return f'ArrayDataType(name={self.name}, element_type={self.element_type})'\n\n def metadata_grid(self) -> List[Union[Decimal, str]]:\n \"\"\"Return the data type as a list, used to populate nested_metadata grid variable.\n\n Returns:\n List with format [format_string, metl_type, metl_precision, metl_scale, snowflake_type]\n \"\"\"\n return [\n '',\n 'VARIANT',\n Decimal(0),\n Decimal(0),\n 'VARIANT',\n ]\n\n\nclass RecordDataType(DataType):\n \"\"\"Class to hold record data type and return the details in various string formats.\"\"\"\n\n def __init__(\n self, field_list: List[Field], type_name: str = '', type_namespace: str = ''\n ) -> None:\n \"\"\"Initialise the class.\"\"\"\n super().__init__(name='record')\n self.field_list = field_list\n if type_namespace:\n self.type_name = f'{type_namespace}.{type_name}'\n else:\n self.type_name = type_name\n self.bytes_to_decimal_function = get_bytes_to_decimal_function()\n\n def __repr__(self) -> str:\n \"\"\"String representation of the class.\"\"\"\n return f'RecordDataType(name={self.name}, type_name={self.type_name}, field_list={self.field_list})'\n\n def metadata_grid(self) -> List[Union[Decimal, str]]:\n \"\"\"Return the data type as a list, used to populate nested_metadata grid variable.\n\n Returns:\n List with format [format_string, metl_type, metl_precision, metl_scale, snowflake_type]\n \"\"\"\n if (\n self.type_name == 'io.debezium.data.VariableScaleDecimal'\n and self.bytes_to_decimal_function\n ):\n\n return [\n f'{self.bytes_to_decimal_function}({{col}}:\"value\"::VARCHAR, {{col}}:\"scale\"::NUMBER, NULL)::VARIANT',\n 'VARIANT',\n Decimal(0),\n Decimal(0),\n 'VARIANT',\n ]\n\n else:\n return [\n '',\n 'VARIANT',\n Decimal(0),\n Decimal(0),\n 'VARIANT',\n ]\n\n\n# Avro schema does not repeat schema for named record types\n# Store each named record type in this dictionary, so we can look up the type if used again\nnamed_record_types: Dict[str, RecordDataType] = dict()\n\n\ndef build_schema_objects(schema: Dict[str, Any]) -> RecordDataType:\n \"\"\"Convert Avro schema into objects.\"\"\"\n if schema.get('type', '') != 'record':\n raise ValueError('Avro schema not in expected format')\n\n schema_objects = get_data_type(schema)\n\n if not isinstance(schema_objects, RecordDataType):\n raise ValueError('Avro schema not in expected format')\n\n return schema_objects\n\n\ndef get_data_type(type_object: Any) -> DataType:\n \"\"\"Convert Avro dictionary object into DataType object.\"\"\"\n # discard any nullable wrapper on type_object\n type_object = extract_from_nullable_union(type_object)\n\n # check if it's a primitive data type\n if isinstance(type_object, str) and type_object in PRIMITIVE_TYPES:\n return DataType(name=type_object)\n\n # check if this is a previously defined record type\n elif isinstance(type_object, str) and type_object in named_record_types:\n return named_record_types[type_object]\n\n # logical types have format {\"type\": \"\", \"logicalType\": \"...\"}\n elif (\n isinstance(type_object, dict)\n and type_object.get('type', '') in PRIMITIVE_TYPES\n and type_object.get('logicalType', '')\n ):\n\n primitive_type = type_object.pop('type')\n logical_type = type_object.pop('logicalType')\n\n return LogicalDataType(\n name=primitive_type, logical_type=logical_type, properties=type_object\n )\n\n # array types have format {\"type\": \"array\", \"items\": \"element_type\"}\n elif isinstance(type_object, dict) and type_object.get('type', '') == 'array':\n return ArrayDataType(element_type=get_data_type(type_object.get('items', '')))\n\n # record types have format {\"type\": \"record\", ... \"fields\": []}\n elif isinstance(type_object, dict) and type_object.get('type', '') == 'record':\n record = RecordDataType(\n field_list=get_fields(type_object),\n type_name=type_object.get('name', ''),\n type_namespace=type_object.get('namespace', ''),\n )\n\n # add record to dictionary of record types\n named_record_types[record.type_name] = record\n return record\n\n # unexpected type\n else:\n return UnknownDataType(name=str(type_object))\n\n\ndef extract_from_nullable_union(type_object: Any) -> Any:\n \"\"\"Nullable types have format ['null', 'type'], extract the actual type.\"\"\"\n if isinstance(type_object, list):\n if len(type_object) == 2 and 'null' in type_object:\n type_object.remove('null')\n type_object = type_object[0]\n\n return type_object\n\n\ndef get_fields(record_object: Dict[str, Any]) -> List[Field]:\n \"\"\"Extract list of Field objects from a record.\"\"\"\n field_list = []\n\n for field in record_object.get('fields', []):\n field_list.append(\n Field(\n name=field.get('name', ''),\n data_type=get_data_type(field.get('type', '')),\n )\n )\n\n return field_list\n\n\ndef trim_strings() -> bool:\n \"\"\"Should we apply trim format function?\"\"\"\n return trim_all_strings == 'y'\n\n\ndef get_bytes_to_decimal_function() -> str:\n \"\"\"Should we apply a function to cast bytes to decimal?\"\"\"\n return bytes_to_decimal_function\n\n\ndef main() -> None:\n \"\"\"Convert Avro schema to nested_metadata grid variable.\"\"\"\n # convert Avro schema as json string to dictionary\n schema_dict = json.loads(avro_schema)\n\n # convert dictionary to list of Field objects\n schema_obj = build_schema_objects(schema_dict)\n\n # format the child fields in the `before` Field object\n nested_metadata = []\n for parent_field in schema_obj.field_list:\n if parent_field.name == 'before' and isinstance(\n parent_field.data_type, RecordDataType\n ):\n\n nested_metadata = [\n [\n field.name,\n field.name.upper(),\n *field.data_type.metadata_grid(), # unpack list of [format_string, type, precision,\n # scale, database_type]\n '', # default_value\n 'No', # not_null\n 'No', # unique\n ]\n for field in parent_field.data_type.field_list\n ]\n\n elif parent_field.name == 'metadata' and isinstance(\n parent_field.data_type, RecordDataType\n ):\n event_metadata = [\n [\n field.name,\n *field.data_type.metadata_grid(), # unpack list of [format_string, type, precision,\n # scale, database_type]\n ]\n for field in parent_field.data_type.field_list\n ]\n\n print('nested_metadata:')\n for row in nested_metadata:\n print(\"\\t\".join(str(col) for col in row))\n context.updateGridVariable('nested_metadata', nested_metadata)\n\n print()\n print('event_metadata:')\n for row in event_metadata:\n print(\"\\t\".join(str(col) for col in row))\n context.updateGridVariable('event_metadata', event_metadata)\n\n\nif __name__ == \"__main__\":\n main()\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41458":{"id":41458,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":1155583855,"x":1035,"y":315,"width":32,"height":32,"inputConnectorIDs":[41504],"outputSuccessConnectorIDs":[41507],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Read primary key from latest metadata in stage table"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Basic/Advanced","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Advanced"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"[Environment Default]"}}}},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Table Columns","elements":{},"visible":false,"status":{},"columnNames":[]},"7":{"slot":7,"name":"SQL Query","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${get_primary_key_sql}"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Grid Variable","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"primary_key"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Grid Variable Mapping","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"KEY_COLUMN"}}}},"visible":true,"status":{},"columnNames":[]},"10":{"slot":10,"name":"Limit","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"100"}}}},"visible":false,"status":{},"columnNames":[]},"11":{"slot":11,"name":"Order By","elements":{},"visible":false,"status":{},"columnNames":[]},"12":{"slot":12,"name":"Sort","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Ascending"}}}},"visible":false,"status":{},"columnNames":[]},"13":{"slot":13,"name":"Filter Conditions","elements":{},"visible":false,"status":{},"columnNames":[]},"14":{"slot":14,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"AND"}}}},"visible":false,"status":{},"columnNames":[]},"20":{"slot":20,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"[Environment Default]"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41459":{"id":41459,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":-80,"y":240,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[41515],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41460":{"id":41460,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":1195,"y":315,"width":32,"height":32,"inputConnectorIDs":[41507],"outputSuccessConnectorIDs":[41508],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Build join expression"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"from pprint import pprint\n\nprimary_key = context.getGridVariable('primary_key')\nprimary_key_override = context.getGridVariable('primary_key_override')\n\nprint('primary_key:')\npprint(primary_key, width=150)\nprint()\n\nprint('primary_key_override:')\npprint(primary_key_override, width=150)\nprint()\n\nif not (primary_key or primary_key_override):\n print(f'Primary key required for {transformation_type!r} transformation type, but no primary '\n 'key detected in change data.\\n'\n \"The transformation type for this table will be updated to 'change log'.\")\n print()\n \n transformation_type = 'change log'\n print(f'transformation_type: {transformation_type}')\n context.updateVariable('transformation_type', transformation_type)\n exit()\n\nif primary_key_override:\n primary_key = [[column.upper()] for *_, column in primary_key_override]\n\n print('primary_key:')\n pprint(primary_key, width=150)\n context.updateGridVariable('primary_key', primary_key)\n print()\n\n# build join expression\npredicates = [f'\"SOURCE\".\"{col[0]}\" = \"TARGET\".\"{col[0]}\"' for col in primary_key]\n\njoin_expression = \" AND \".join(predicates)\n\nprint(f'join_expression: {join_expression}')\ncontext.updateVariable('join_expression', join_expression)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41462":{"id":41462,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":875,"y":315,"width":32,"height":32,"inputConnectorIDs":[41511],"outputSuccessConnectorIDs":[41504],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Build get_primary_key_sql"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"get_primary_key_sql = \\\nf'''WITH \n\"METADATA_KEYS\" AS (\n SELECT \n \"VALUE\":\"metadata\".\"key\" AS \"KEYS\"\n FROM\n \"{target_database}\".\"{stage_schema}\".\"{stage_table}\"\n WHERE \n \"DATABASE\" = '{source_database}' \n AND \"SCHEMA\" = '{source_schema}' \n AND \"TABLE\" = '{source_table}'\n AND \"VERSION\" = {source_version}\n AND \"DATE_HOUR\" >= '{min_date_hour}'\n AND \"FILENAME\" >= '{min_filename}'\n LIMIT 1\n)\nSELECT\n UPPER(\"KEY_LIST\".\"VALUE\"::VARCHAR) as \"KEY_COLUMN\"\nFROM \"METADATA_KEYS\",\nLATERAL FLATTEN (input => \"METADATA_KEYS\".\"KEYS\", recursive => FALSE) \"KEY_LIST\"'''\n\nprint(f'get_primary_key_sql: \\n{get_primary_key_sql}')\ncontext.updateVariable('get_primary_key_sql', get_primary_key_sql)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41463":{"id":41463,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":715,"y":235,"width":32,"height":32,"inputConnectorIDs":[44683],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[41510],"outputFalseConnectorIDs":[41511],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Transformation is Change Log?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"transformation_type"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"change log"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41464":{"id":41464,"inputCardinality":"MANY","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1343684451,"x":1355,"y":235,"width":32,"height":32,"inputConnectorIDs":[41508,41510],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[41513],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Join paths (1)"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41465":{"id":41465,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":1760,"y":240,"width":32,"height":32,"inputConnectorIDs":[41506],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[41509],"outputFalseConnectorIDs":[41512],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Target table exists?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"target_table_exists"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Greater than"},"4":{"slot":4,"type":"STRING","value":"0"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41466":{"id":41466,"inputCardinality":"MANY","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1343684451,"x":2240,"y":240,"width":32,"height":32,"inputConnectorIDs":[61644,41501],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[47167],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Join paths (2)"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41467":{"id":41467,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":1920,"y":240,"width":32,"height":32,"inputConnectorIDs":[41509],"outputSuccessConnectorIDs":[41501],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Resolve metadata drift"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync Single Table - 1-2-4 - Resolve metadata drift"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"target_database"},"2":{"slot":2,"type":"STRING","value":"${target_database}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"target_schema"},"2":{"slot":2,"type":"STRING","value":"${target_schema}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"target_table"},"2":{"slot":2,"type":"STRING","value":"${target_table}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"schema_drift_action"},"2":{"slot":2,"type":"STRING","value":"${schema_drift_action}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"connector"},"2":{"slot":2,"type":"STRING","value":"${connector}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"commit_id_name"},"2":{"slot":2,"type":"STRING","value":"${commit_id_name}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"target_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"column_name"},"2":{"slot":2,"type":"GRID","value":"type"},"3":{"slot":3,"type":"GRID","value":"size"},"4":{"slot":4,"type":"GRID","value":"precision"},"5":{"slot":5,"type":"GRID","value":"database_type"},"6":{"slot":6,"type":"GRID","value":"default_value"},"7":{"slot":7,"type":"GRID","value":"not_null"},"8":{"slot":8,"type":"GRID","value":"unique"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"latest_target_metadata"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync Single Table (Exchange)"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Data Loader.CDC"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"3"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41468":{"id":41468,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":1600,"y":240,"width":32,"height":32,"inputConnectorIDs":[41513],"outputSuccessConnectorIDs":[41506],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"gridExportMappings":{"1":{"slot":1,"fromGrid":"target_metadata","toGrid":"target_metadata","columnMapToFrom":{"database_type":"database_type","size":"size","not_null":"not_null","unique":"unique","precision":"precision","column_name":"column_name","default_value":"default_value","type":"type"},"validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Build target table metadata from nested_metadata"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync Single Table - 1-2-3 - Build new target table metadata"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"append_metadata"},"2":{"slot":2,"type":"STRING","value":"${append_metadata}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"transformation_type"},"2":{"slot":2,"type":"STRING","value":"${transformation_type}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"commit_id_datatype"},"2":{"slot":2,"type":"STRING","value":"${commit_id_datatype}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"commit_id_precision"},"2":{"slot":2,"type":"STRING","value":"${commit_id_precision}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"commit_id_scale"},"2":{"slot":2,"type":"STRING","value":"${commit_id_scale}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"commit_id_database_type"},"2":{"slot":2,"type":"STRING","value":"${commit_id_database_type}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"nested_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"property"},"2":{"slot":2,"type":"GRID","value":"alias"},"3":{"slot":3,"type":"GRID","value":"format_string"},"4":{"slot":4,"type":"GRID","value":"type"},"5":{"slot":5,"type":"GRID","value":"size"},"6":{"slot":6,"type":"GRID","value":"precision"},"7":{"slot":7,"type":"GRID","value":"database_type"},"8":{"slot":8,"type":"GRID","value":"default_value"},"9":{"slot":9,"type":"GRID","value":"not_null"},"10":{"slot":10,"type":"GRID","value":"unique"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"nested_metadata"},"2":{"slot":2,"type":"STRING","value":"grid"}}},"2":{"slot":2,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"primary_key"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"key_column"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"primary_key"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync Single Table (Exchange)"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Data Loader.CDC"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"3"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41469":{"id":41469,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":2640,"y":240,"width":32,"height":32,"inputConnectorIDs":[47195],"outputSuccessConnectorIDs":[46006],"outputFailureConnectorIDs":[41517],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"logged_rows_deleted","mapTo":"logged_rows_deleted","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"logged_rows_updated","mapTo":"logged_rows_updated","validationStatus":null,"validationMessage":null},"3":{"slot":3,"fromId":null,"fromName":"logged_rows_inserted","mapTo":"logged_rows_inserted","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Transform and Load"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync Single Table - 1-2-5 - Transform and Load"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"append_metadata"},"2":{"slot":2,"type":"STRING","value":"${append_metadata}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"commit_id_name"},"2":{"slot":2,"type":"STRING","value":"${commit_id_name}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"connector"},"2":{"slot":2,"type":"STRING","value":"${connector}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"commit_id_datatype"},"2":{"slot":2,"type":"STRING","value":"${commit_id_datatype}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"commit_id_precision"},"2":{"slot":2,"type":"STRING","value":"${commit_id_precision}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"commit_id_scale"},"2":{"slot":2,"type":"STRING","value":"${commit_id_scale}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"commit_id_database_type"},"2":{"slot":2,"type":"STRING","value":"${commit_id_database_type}"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"stage_schema"},"2":{"slot":2,"type":"STRING","value":"${stage_schema}"}}},"9":{"slot":9,"values":{"1":{"slot":1,"type":"STRING","value":"stage_table"},"2":{"slot":2,"type":"STRING","value":"${stage_table}"}}},"10":{"slot":10,"values":{"1":{"slot":1,"type":"STRING","value":"join_expression"},"2":{"slot":2,"type":"STRING","value":"${join_expression}"}}},"11":{"slot":11,"values":{"1":{"slot":1,"type":"STRING","value":"min_date_hour"},"2":{"slot":2,"type":"STRING","value":"${min_date_hour}"}}},"12":{"slot":12,"values":{"1":{"slot":1,"type":"STRING","value":"min_filename"},"2":{"slot":2,"type":"STRING","value":"${min_filename}"}}},"13":{"slot":13,"values":{"1":{"slot":1,"type":"STRING","value":"source_database"},"2":{"slot":2,"type":"STRING","value":"${source_database}"}}},"14":{"slot":14,"values":{"1":{"slot":1,"type":"STRING","value":"source_schema"},"2":{"slot":2,"type":"STRING","value":"${source_schema}"}}},"15":{"slot":15,"values":{"1":{"slot":1,"type":"STRING","value":"source_table"},"2":{"slot":2,"type":"STRING","value":"${source_table}"}}},"16":{"slot":16,"values":{"1":{"slot":1,"type":"STRING","value":"source_version"},"2":{"slot":2,"type":"STRING","value":"${source_version}"}}},"17":{"slot":17,"values":{"1":{"slot":1,"type":"STRING","value":"warehouse"},"2":{"slot":2,"type":"STRING","value":"${warehouse}"}}},"18":{"slot":18,"values":{"1":{"slot":1,"type":"STRING","value":"target_database"},"2":{"slot":2,"type":"STRING","value":"${target_database}"}}},"19":{"slot":19,"values":{"1":{"slot":1,"type":"STRING","value":"target_schema"},"2":{"slot":2,"type":"STRING","value":"${target_schema}"}}},"20":{"slot":20,"values":{"1":{"slot":1,"type":"STRING","value":"target_table"},"2":{"slot":2,"type":"STRING","value":"${target_table}"}}},"21":{"slot":21,"values":{"1":{"slot":1,"type":"STRING","value":"transformation_type"},"2":{"slot":2,"type":"STRING","value":"${transformation_type}"}}},"22":{"slot":22,"values":{"1":{"slot":1,"type":"STRING","value":"logging_enabled"},"2":{"slot":2,"type":"STRING","value":"${logging_enabled}"}}},"23":{"slot":23,"values":{"1":{"slot":1,"type":"STRING","value":"log_table"},"2":{"slot":2,"type":"STRING","value":"${log_table}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"nested_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"property"},"2":{"slot":2,"type":"GRID","value":"alias"},"3":{"slot":3,"type":"GRID","value":"format_string"},"4":{"slot":4,"type":"GRID","value":"type"},"5":{"slot":5,"type":"GRID","value":"size"},"6":{"slot":6,"type":"GRID","value":"precision"},"7":{"slot":7,"type":"GRID","value":"database_type"},"8":{"slot":8,"type":"GRID","value":"default_value"},"9":{"slot":9,"type":"GRID","value":"not_null"},"10":{"slot":10,"type":"GRID","value":"unique"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"nested_metadata"},"2":{"slot":2,"type":"STRING","value":"grid"}}},"2":{"slot":2,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"primary_key"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"key_column"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"primary_key"},"2":{"slot":2,"type":"STRING","value":"grid"}}},"3":{"slot":3,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"target_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"column_name"},"2":{"slot":2,"type":"GRID","value":"type"},"3":{"slot":3,"type":"GRID","value":"size"},"4":{"slot":4,"type":"GRID","value":"precision"},"5":{"slot":5,"type":"GRID","value":"database_type"},"6":{"slot":6,"type":"GRID","value":"default_value"},"7":{"slot":7,"type":"GRID","value":"not_null"},"8":{"slot":8,"type":"GRID","value":"unique"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"target_metadata"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync Single Table (Exchange)"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Data Loader.CDC"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"3"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41470":{"id":41470,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":1611478312,"x":1920,"y":320,"width":32,"height":32,"inputConnectorIDs":[41512],"outputSuccessConnectorIDs":[61618],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Create target table"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"New Table Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"target_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"column_name"},"2":{"slot":2,"type":"GRID","value":"type"},"3":{"slot":3,"type":"GRID","value":"size"},"4":{"slot":4,"type":"GRID","value":"precision"},"5":{"slot":5,"type":"GRID","value":"default_value"},"6":{"slot":6,"type":"GRID","value":"not_null"},"7":{"slot":7,"type":"GRID","value":"unique"},"8":{"slot":8,"type":"GRID","value":null}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Create/Replace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Create"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Clustering Keys","elements":{},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Data Retention Time in Days","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":""}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Comment","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"11":{"slot":11,"name":"Primary Keys","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"primary_key"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"key_column"}}}},"visible":true,"status":{},"columnNames":[]},"12":{"slot":12,"name":"Table Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Permanent"}}}},"visible":true,"status":{},"columnNames":[]},"13":{"slot":13,"name":"Default DDL Collation","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44649":{"id":44649,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":480,"y":240,"width":32,"height":32,"inputConnectorIDs":[44650],"outputSuccessConnectorIDs":[44683],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"commit_id_name","mapTo":"commit_id_name","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"commit_id_datatype","mapTo":"commit_id_datatype","validationStatus":null,"validationMessage":null},"3":{"slot":3,"fromId":null,"fromName":"commit_id_precision","mapTo":"commit_id_precision","validationStatus":null,"validationMessage":null},"4":{"slot":4,"fromId":null,"fromName":"commit_id_scale","mapTo":"commit_id_scale","validationStatus":null,"validationMessage":null},"5":{"slot":5,"fromId":null,"fromName":"commit_id_database_type","mapTo":"commit_id_database_type","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Validate event metadata"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync Single Table - 1-2-2 - Validate event metadata"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"connector"},"2":{"slot":2,"type":"STRING","value":"${connector}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"commit_id_name"},"2":{"slot":2,"type":"STRING","value":"${commit_id_name}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"commit_id_datatype"},"2":{"slot":2,"type":"STRING","value":"${commit_id_datatype}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"commit_id_precision"},"2":{"slot":2,"type":"STRING","value":"${commit_id_precision}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"commit_id_scale"},"2":{"slot":2,"type":"STRING","value":"${commit_id_scale}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"commit_id_database_type"},"2":{"slot":2,"type":"STRING","value":"${commit_id_database_type}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"event_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"property"},"2":{"slot":2,"type":"GRID","value":"format_string"},"3":{"slot":3,"type":"GRID","value":"type"},"4":{"slot":4,"type":"GRID","value":"size"},"5":{"slot":5,"type":"GRID","value":"precision"},"6":{"slot":6,"type":"GRID","value":"database_type"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"event_metadata"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync Single Table (Exchange)"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Data Loader.CDC"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"3"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"45316":{"id":45316,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":3040,"y":320,"width":32,"height":32,"inputConnectorIDs":[45319],"outputSuccessConnectorIDs":[45322],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Ensure schema.avsc file from previous version has been tagged"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"from decimal import Decimal\n\n\nfile_list = context.getGridVariable('file_list')\n\nfirst_file = file_list[0][1]\ntable_prefix = first_file.rsplit('/', 6)[0]\n\nprevious_version = source_version - 1\nprevious_schema_filepath = f'{table_prefix}/version={previous_version}/schema.avsc'\nprint(f'previous_schema_filepath: {previous_schema_filepath}')\nprint()\n\nprint('Adding previous version schema.avsc to tag list to ensure it has been tagged.')\n\nfile_list.append(\n [previous_version, previous_schema_filepath]\n)\ncontext.updateGridVariable('file_list', file_list)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"45980":{"id":45980,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":2880,"y":240,"width":32,"height":32,"inputConnectorIDs":[46006],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[45319],"outputFalseConnectorIDs":[45985],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Tag processed files?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"tag_processed_files"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"y"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"45981":{"id":45981,"inputCardinality":"MANY","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1343684451,"x":3344,"y":240,"width":32,"height":32,"inputConnectorIDs":[45985,45983],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[46082],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Join paths (3)"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"45982":{"id":45982,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":3200,"y":320,"width":32,"height":32,"inputConnectorIDs":[45322],"outputSuccessConnectorIDs":[45983],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Apply tags"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync Single Table - 1-2-6 - Apply tags"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"cloud_storage_url"},"2":{"slot":2,"type":"STRING","value":"${cloud_storage_url}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"azure_key_vault_url"},"2":{"slot":2,"type":"STRING","value":"${azure_key_vault_url}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"storage_account_key_secret"},"2":{"slot":2,"type":"STRING","value":"${storage_account_key_secret}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"file_list"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"filepath"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"processed_files"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync Single Table (Exchange)"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Data Loader.CDC"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"3"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"46022":{"id":46022,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":2480,"y":240,"width":32,"height":32,"inputConnectorIDs":[47167],"outputSuccessConnectorIDs":[47195],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Filter file_list for current version"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"file_list = context.getGridVariable('file_list') or []\n\nfile_list = [[version, filepath] for version, filepath in file_list if version == source_version]\n\nprint('file_list:')\nfor row in file_list:\n print(\"\\t\".join(str(col) for col in row))\ncontext.updateGridVariable('file_list', file_list)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"61617":{"id":61617,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":2080,"y":320,"width":32,"height":32,"inputConnectorIDs":[61618],"outputSuccessConnectorIDs":[61644],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Set target_table_exists"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"from decimal import Decimal\n\n\ntarget_table_exists = Decimal(1)\n\nprint(f'target_table_exists: {target_table_exists}')\ncontext.updateVariable('target_table_exists', target_table_exists)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"41471":{"id":41471,"sourceID":41456,"targetID":41457},"41501":{"id":41501,"sourceID":41467,"targetID":41466},"41504":{"id":41504,"sourceID":41462,"targetID":41458},"41506":{"id":41506,"sourceID":41468,"targetID":41465},"41507":{"id":41507,"sourceID":41458,"targetID":41460},"41508":{"id":41508,"sourceID":41460,"targetID":41464},"44650":{"id":44650,"sourceID":41457,"targetID":44649},"44683":{"id":44683,"sourceID":44649,"targetID":41463},"45322":{"id":45322,"sourceID":45316,"targetID":45982},"45983":{"id":45983,"sourceID":45982,"targetID":45981},"46006":{"id":46006,"sourceID":41469,"targetID":45980},"47195":{"id":47195,"sourceID":46022,"targetID":41469},"61618":{"id":61618,"sourceID":41470,"targetID":61617},"61644":{"id":61644,"sourceID":61617,"targetID":41466}},"failureConnectors":{"41517":{"id":41517,"sourceID":41469,"targetID":41454},"41518":{"id":41518,"sourceID":41457,"targetID":41455}},"unconditionalConnectors":{"41513":{"id":41513,"sourceID":41464,"targetID":41468},"41515":{"id":41515,"sourceID":41459,"targetID":41456},"46082":{"id":46082,"sourceID":45981,"targetID":41453},"47167":{"id":47167,"sourceID":41466,"targetID":46022}},"trueConnectors":{"41509":{"id":41509,"sourceID":41465,"targetID":41467},"41510":{"id":41510,"sourceID":41463,"targetID":41464},"45319":{"id":45319,"sourceID":45980,"targetID":45316}},"falseConnectors":{"41511":{"id":41511,"sourceID":41463,"targetID":41462},"41512":{"id":41512,"sourceID":41465,"targetID":41470},"45985":{"id":45985,"sourceID":45980,"targetID":45981}},"iterationConnectors":{},"noteConnectors":{},"canUndo":false,"undoCommand":"","undoCreated":0,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"41451":{"id":41451,"x":1526,"y":85,"width":788,"height":310,"text":"**Prepare target table**\n\nBuild __target_metadata__ from the Avro schema for the current version\n\nIf the target table exists, check for schema drift\n\nElse create the target table using __target_metadata__\n","colour":"e6e63c"},"41452":{"id":41452,"x":2406,"y":84,"width":310,"height":232,"text":"**Transform and load new data from stage table into target table**\n\nNote: file_list is only used if we are applying tags after the data has been loaded. However, we are printing out here for observability into which files the transformation job will be processing.","colour":"e6e63c"},"41499":{"id":41499,"x":6,"y":85,"width":309,"height":230,"text":"**Parse Avro schema**\n\nParse the __avro_schema__ to extract\n\n* __nested_metadata__grid var, used to create target table and check for schema drift\n","colour":"e6e63c"},"41500":{"id":41500,"x":646,"y":5,"width":791,"height":389,"text":"**Read primary key**\n\nNote: the primary key and join expression are not required for a change log. In particular, we do not want to add a primary key to the target table for a change log as we will have multiple events for each key.\n\nRead the primary key from data in the latest __date_hour__, and store in __primary_key__ grid variable.\n\nUse the __primary_key__ grid variable to build the Join Expression for the Table Update components in the Transformation job.\n\nThe join expression is required to merge changes into the target table. So If no primary key is detected, raise an error and stop the job.","colour":"e6e63c"},"43845":{"id":43845,"x":405,"y":84,"width":149,"height":232,"text":"**Check event metadata**\n\nValidate that the change event metadata contains the required fields for this version of the shared job","colour":"e6e63c"},"45986":{"id":45986,"x":2806,"y":87,"width":628,"height":306,"text":"**Tag processed files**\n\nAfter Avro files have been loaded, apply a tag to the files so that lifecycle policies can be applied to archive them.","colour":"e6e63c"}},"variables":{"connector":{"definition":{"name":"connector","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"log_table":{"definition":{"name":"log_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"warehouse":{"definition":{"name":"warehouse","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"avro_schema":{"definition":{"name":"avro_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"stage_table":{"definition":{"name":"stage_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"min_filename":{"definition":{"name":"min_filename","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"source_table":{"definition":{"name":"source_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"stage_schema":{"definition":{"name":"stage_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_table":{"definition":{"name":"target_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"min_date_hour":{"definition":{"name":"min_date_hour","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"source_schema":{"definition":{"name":"source_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_schema":{"definition":{"name":"target_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"commit_id_name":{"definition":{"name":"commit_id_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"source_version":{"definition":{"name":"source_version","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"append_metadata":{"definition":{"name":"append_metadata","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"commit_id_scale":{"definition":{"name":"commit_id_scale","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"join_expression":{"definition":{"name":"join_expression","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"logging_enabled":{"definition":{"name":"logging_enabled","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":null},"source_database":{"definition":{"name":"source_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_database":{"definition":{"name":"target_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"trim_all_strings":{"definition":{"name":"trim_all_strings","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"cloud_storage_url":{"definition":{"name":"cloud_storage_url","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"commit_id_datatype":{"definition":{"name":"commit_id_datatype","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"azure_key_vault_url":{"definition":{"name":"azure_key_vault_url","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"commit_id_precision":{"definition":{"name":"commit_id_precision","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"get_primary_key_sql":{"definition":{"name":"get_primary_key_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"logged_rows_deleted":{"definition":{"name":"logged_rows_deleted","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"0"},"logged_rows_updated":{"definition":{"name":"logged_rows_updated","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"0"},"schema_drift_action":{"definition":{"name":"schema_drift_action","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"tag_processed_files":{"definition":{"name":"tag_processed_files","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_table_exists":{"definition":{"name":"target_table_exists","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"transformation_type":{"definition":{"name":"transformation_type","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"logged_rows_inserted":{"definition":{"name":"logged_rows_inserted","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"0"},"commit_id_database_type":{"definition":{"name":"commit_id_database_type","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"bytes_to_decimal_function":{"definition":{"name":"bytes_to_decimal_function","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"storage_account_key_secret":{"definition":{"name":"storage_account_key_secret","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""}},"grids":{"file_list":{"definition":{"name":"file_list","scope":"TASKBATCH","definitions":[{"name":"version","type":"DECIMAL"},{"name":"filepath","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]},"primary_key":{"definition":{"name":"primary_key","scope":"TASKBATCH","definitions":[{"name":"key_column","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[]},"event_metadata":{"definition":{"name":"event_metadata","scope":"TASKBATCH","definitions":[{"name":"property","type":"TEXT"},{"name":"format_string","type":"TEXT"},{"name":"type","type":"TEXT"},{"name":"size","type":"DECIMAL"},{"name":"precision","type":"DECIMAL"},{"name":"database_type","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[]},"nested_metadata":{"definition":{"name":"nested_metadata","scope":"TASKBATCH","definitions":[{"name":"property","type":"TEXT"},{"name":"alias","type":"TEXT"},{"name":"format_string","type":"TEXT"},{"name":"type","type":"TEXT"},{"name":"size","type":"DECIMAL"},{"name":"precision","type":"DECIMAL"},{"name":"database_type","type":"TEXT"},{"name":"default_value","type":"TEXT"},{"name":"not_null","type":"TEXT"},{"name":"unique","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[]},"target_metadata":{"definition":{"name":"target_metadata","scope":"TASKBATCH","definitions":[{"name":"column_name","type":"TEXT"},{"name":"type","type":"TEXT"},{"name":"size","type":"DECIMAL"},{"name":"precision","type":"DECIMAL"},{"name":"database_type","type":"TEXT"},{"name":"default_value","type":"TEXT"},{"name":"not_null","type":"TEXT"},{"name":"unique","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[]},"primary_key_override":{"definition":{"name":"primary_key_override","scope":"TASKBATCH","definitions":[{"name":"source_database","type":"TEXT"},{"name":"source_schema","type":"TEXT"},{"name":"source_table","type":"TEXT"},{"name":"source_column","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]}}},"Sync Single Table - 1-0 - Validate bytes to decimal function":{"id":-2,"revision":1,"created":1728917529394,"timestamp":1728917529394,"components":{"43100":{"id":43100,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":0,"y":0,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[43133],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Start","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"43132":{"id":43132,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":160,"y":0,"width":32,"height":32,"inputConnectorIDs":[43133],"outputSuccessConnectorIDs":[43187],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Fully qualify user defined function name"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"def unquote_identifier(identifier: str) -> str:\n if identifier[0] == '\"' and identifier[-1] == '\"':\n return identifier[1:-1]\n else:\n return identifier\n\n\nprint(f'initial bytes_to_decimal_function: {bytes_to_decimal_function!r}')\nprint()\n\nif not bytes_to_decimal_function:\n print(\n 'No function specified for casting bytes to decimals. \\n'\n 'Any VariableScaleDecimal columns in the Avro files will be left '\n 'as byte arrays in variant columns in the target table.'\n )\n exit()\n\nudf_list = bytes_to_decimal_function.split('.', 2)\n\nif len(udf_list) == 1:\n print('No database or schema specified for function, so using environment default.')\n print()\n function_database = environment_database\n function_schema = environment_default_schema\n function_name = unquote_identifier(udf_list[0])\nelif len(udf_list) == 2:\n print('No database specified for function, so using environment default.')\n print()\n function_database = environment_database\n function_schema = unquote_identifier(udf_list[0])\n function_name = unquote_identifier(udf_list[1])\nelse:\n function_database = unquote_identifier(udf_list[0])\n function_schema = unquote_identifier(udf_list[1])\n function_name = unquote_identifier(udf_list[2])\n \nprint(f'function_database: {function_database!r}')\ncontext.updateVariable('function_database', function_database)\n\nprint(f'function_schema: {function_schema!r}')\ncontext.updateVariable('function_schema', function_schema)\n\nprint(f'function_name: {function_name!r}')\ncontext.updateVariable('function_name', function_name)\n\nbytes_to_decimal_function = f'\"{function_database}\".\"{function_schema}\".\"{function_name}\"'\n\nprint()\nprint(f'fully qualified bytes_to_decimal_function: {bytes_to_decimal_function!r}')\ncontext.updateVariable('bytes_to_decimal_function', bytes_to_decimal_function)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"43152":{"id":43152,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":556,"y":1,"width":32,"height":32,"inputConnectorIDs":[43231],"outputSuccessConnectorIDs":[43241],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Build check_udf_sql"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"check_udf_sql = \\\nf\"\"\"SELECT \n \"FUNCTION_NAME\",\n \"ARGUMENT_SIGNATURE\",\n \"DATA_TYPE\",\n \"FUNCTION_LANGUAGE\",\n \"RUNTIME_VERSION\"\nFROM \n \"{function_database}\".\"INFORMATION_SCHEMA\".\"FUNCTIONS\"\nWHERE\n \"FUNCTION_CATALOG\" = '{function_database}'\n AND \"FUNCTION_SCHEMA\" = '{function_schema}'\n AND \"FUNCTION_NAME\" = '{function_name}'\"\"\"\n\nprint(f'check_udf_sql: \\n{check_udf_sql}')\ncontext.updateVariable('check_udf_sql', check_udf_sql)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"43154":{"id":43154,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":877,"y":3,"width":32,"height":32,"inputConnectorIDs":[43274],"outputSuccessConnectorIDs":[43192],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Validate user defined function"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"from collections import namedtuple\nfrom decimal import Decimal\nfrom pprint import pprint\n\n\ndef current_runtime(runtime_version, minimum_runtime) -> bool:\n runtime_major, runtime_minor = runtime_version.split('.')\n minimum_major, minimum_minor = minimum_runtime.split('.')\n \n if int(runtime_major) > int(minimum_major):\n return True\n \n if int(runtime_major) == int(minimum_major) and int(runtime_minor) >= int(minimum_minor):\n return True\n \n return False\n\n\nUserDefinedFunction = namedtuple('UserDefinedFunction', 'name argument_signature return_type function_language, runtime_version')\nfunction_found = Decimal(0)\n\nrequired_signature = '(HEXSTRING VARCHAR, SCALE NUMBER, PRECISION NUMBER)'\nrequired_return_type = 'VARCHAR(16777216)'\nrequired_language = 'PYTHON'\nminimum_runtime = '3.11'\n\n\nuser_defined_functions = context.getGridVariable('user_defined_functions') or []\nprint('user_defined_functions:')\npprint(user_defined_functions, width=200)\nprint()\n\nif not user_defined_functions:\n print(f'Unable to find any user defined functions named {bytes_to_decimal_function}')\n exit()\n\nfor user_defined_function in user_defined_functions:\n udf = UserDefinedFunction(*user_defined_function)\n \n if (\n udf.argument_signature == required_signature\n and udf.return_type == required_return_type\n and udf.function_language == required_language\n and current_runtime(udf.runtime_version, minimum_runtime)\n ):\n\n print(\n f'Found user defined function named {bytes_to_decimal_function} '\n f\"with required signature {required_signature!r}, \"\n f\"return type {required_return_type!r} in {required_language!r} {udf.runtime_version}.\"\n )\n print()\n \n function_found = Decimal(1)\n print(f'function_found: {function_found}')\n context.updateVariable('function_found', function_found)\n \n break\n\nelse:\n print(\n f'Unable to find user defined function named {bytes_to_decimal_function} '\n f\"with required signature {required_signature!r}, \"\n f\"return type {required_return_type!r} in a minimum of {required_language!r} {minimum_runtime}.\"\n )\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"43156":{"id":43156,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":1273,"y":84,"width":32,"height":32,"inputConnectorIDs":[43347],"outputSuccessConnectorIDs":[43356],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Build create_function_sql"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"create_function_sql = \\\nf\"\"\"CREATE OR REPLACE FUNCTION {bytes_to_decimal_function}(HEXSTRING VARCHAR, SCALE NUMBER, PRECISION NUMBER)\nRETURNS VARCHAR\nLANGUAGE PYTHON\nRUNTIME_VERSION = '3.11'\nHANDLER = 'bytes_to_decimal_py'\nas\n$$\ndef bytes_to_decimal_py(hexstring, scale, precision):\n from decimal import Decimal, localcontext\n\n if hexstring is None or scale is None:\n return\n\n int_value = int.from_bytes(\n bytes.fromhex(hexstring),\n byteorder='big',\n signed=True,\n )\n\n if precision is None:\n precision = len(\"%i\" % int_value)\n\n with localcontext() as ctx:\n ctx.prec = precision\n return str(int_value * (Decimal(10) ** -scale))\n$$;\"\"\"\n\nprint(f'create_function_sql: \\n{create_function_sql}')\ncontext.updateVariable('create_function_sql', create_function_sql)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"43158":{"id":43158,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":1600,"y":160,"width":32,"height":32,"inputConnectorIDs":[43373],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[43383],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Check error message"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"print(\n 'Error creating user defined function used to convert VariableScaleDecimal columns.\\n'\n 'If the current role does not have the \"CREATE FUNCTION\" privilege on the schema \\n'\n f'\"{function_database}\".\"{function_schema}\", you should create the function in\\n'\n 'the Snowflake Console using a role with the required privileges.'\n)\nprint()\nprint('Error message:')\nprint(error_message)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"43186":{"id":43186,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":320,"y":0,"width":32,"height":32,"inputConnectorIDs":[43187],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[43231],"outputFalseConnectorIDs":[43210],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"bytes_to_decimal_function parameter supplied?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"bytes_to_decimal_function"},"2":{"slot":2,"type":"STRING","value":"Not"},"3":{"slot":3,"type":"STRING","value":"Blank"},"4":{"slot":4,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"43191":{"id":43191,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":1120,"y":0,"width":32,"height":32,"inputConnectorIDs":[43192],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[44634],"outputFalseConnectorIDs":[43347],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"User defined function found?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"function_found"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Greater than"},"4":{"slot":4,"type":"STRING","value":"0"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"43207":{"id":43207,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":320,"y":156,"width":32,"height":32,"inputConnectorIDs":[43210],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Success - No bytes_to_decimal_function parameter supplied"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"43212":{"id":43212,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":1760,"y":0,"width":32,"height":32,"inputConnectorIDs":[44641],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Success"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"43240":{"id":43240,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":1155583855,"x":720,"y":0,"width":32,"height":32,"inputConnectorIDs":[43241],"outputSuccessConnectorIDs":[43274],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Lookup user defined function"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Basic / Advanced","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Advanced"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"[Environment Default]"}}}},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Table Columns","elements":{},"visible":false,"status":{},"columnNames":[]},"7":{"slot":7,"name":"SQL Query","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${check_udf_sql}"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Grid Variable","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"user_defined_functions"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Grid Variable Mapping","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"FUNCTION_NAME"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"ARGUMENT_SIGNATURE"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"DATA_TYPE"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"FUNCTION_LANGUAGE"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"RUNTIME_VERSION"}}}},"visible":true,"status":{},"columnNames":[]},"10":{"slot":10,"name":"Limit","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"100"}}}},"visible":false,"status":{},"columnNames":[]},"11":{"slot":11,"name":"Order By","elements":{},"visible":false,"status":{},"columnNames":[]},"12":{"slot":12,"name":"Sort","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Ascending"}}}},"visible":false,"status":{},"columnNames":[]},"13":{"slot":13,"name":"Filter Conditions","elements":{},"visible":false,"status":{},"columnNames":[]},"14":{"slot":14,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"AND"}}}},"visible":false,"status":{},"columnNames":[]},"20":{"slot":20,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"[Environment Default]"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"43351":{"id":43351,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-798585337,"x":1440,"y":80,"width":32,"height":32,"inputConnectorIDs":[43356],"outputSuccessConnectorIDs":[44637],"outputFailureConnectorIDs":[43373],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"Message","mapTo":"error_message","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Create user defined function"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"SQL Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${create_function_sql}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"43375":{"id":43375,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":1760,"y":160,"width":32,"height":32,"inputConnectorIDs":[43383],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Failure"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44626":{"id":44626,"inputCardinality":"MANY","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1343684451,"x":1600,"y":0,"width":32,"height":32,"inputConnectorIDs":[44634,44637],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[44641],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Join paths"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"43187":{"id":43187,"sourceID":43132,"targetID":43186},"43192":{"id":43192,"sourceID":43154,"targetID":43191},"43241":{"id":43241,"sourceID":43152,"targetID":43240},"43274":{"id":43274,"sourceID":43240,"targetID":43154},"43356":{"id":43356,"sourceID":43156,"targetID":43351},"44637":{"id":44637,"sourceID":43351,"targetID":44626}},"failureConnectors":{"43373":{"id":43373,"sourceID":43351,"targetID":43158}},"unconditionalConnectors":{"43133":{"id":43133,"sourceID":43100,"targetID":43132},"43383":{"id":43383,"sourceID":43158,"targetID":43375},"44641":{"id":44641,"sourceID":44626,"targetID":43212}},"trueConnectors":{"43231":{"id":43231,"sourceID":43186,"targetID":43152},"44634":{"id":44634,"sourceID":43191,"targetID":44626}},"falseConnectors":{"43210":{"id":43210,"sourceID":43186,"targetID":43207},"43347":{"id":43347,"sourceID":43191,"targetID":43156}},"iterationConnectors":{},"noteConnectors":{},"canUndo":false,"undoCommand":"","undoCreated":0,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"43221":{"id":43221,"x":86,"y":-154,"width":307,"height":228,"text":"**Check bytes_to_decimal_function variable**","colour":"e6e63c"},"43276":{"id":43276,"x":486,"y":-155,"width":468,"height":229,"text":"**Check if UDF already exists**\n\nIf a UDF is found with the given name, check that its signature has the correct number of arguments and the correct types.","colour":"e6e63c"},"43390":{"id":43390,"x":1047,"y":-154,"width":627,"height":389,"text":"**If UDF not found, try to create it**","colour":"e6e63c"}},"variables":{"check_udf_sql":{"definition":{"name":"check_udf_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"error_message":{"definition":{"name":"error_message","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"function_name":{"definition":{"name":"function_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"function_found":{"definition":{"name":"function_found","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"0"},"function_schema":{"definition":{"name":"function_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"function_database":{"definition":{"name":"function_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"create_function_sql":{"definition":{"name":"create_function_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"bytes_to_decimal_function":{"definition":{"name":"bytes_to_decimal_function","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""}},"grids":{"user_defined_functions":{"definition":{"name":"user_defined_functions","scope":"TASKBATCH","definitions":[{"name":"function_name","type":"TEXT"},{"name":"argument_signature","type":"TEXT"},{"name":"data_type","type":"TEXT"},{"name":"function_language","type":"TEXT"},{"name":"runtime_version","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[]}}},"Sync Single Table - 1 - Orchestrate":{"id":-2,"revision":1,"created":1728917529394,"timestamp":1728917529394,"components":{"41421":{"id":41421,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":880,"y":160,"width":32,"height":32,"inputConnectorIDs":[59921],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[59959],"outputFalseConnectorIDs":[41478],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"New files found?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"new_files_found"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Greater than or equal to"},"4":{"slot":4,"type":"STRING","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41422":{"id":41422,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":880,"y":320,"width":32,"height":32,"inputConnectorIDs":[41478],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Success - No new data for source table"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41423":{"id":41423,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":-2480,"y":160,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[50229],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41424":{"id":41424,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1032749985,"x":-640,"y":160,"width":32,"height":32,"inputConnectorIDs":[41444],"outputSuccessConnectorIDs":[41441],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Check target table exists"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"SQL Query","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_exists_sql}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Scalar Variable Mapping","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"target_table_exists"},"2":{"slot":2,"type":"STRING","value":"TABLE_EXISTS"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"[Environment Default]"}}}},"visible":false,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Table Columns","elements":{},"visible":false,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"[Environment Default]"}}}},"visible":false,"status":{},"columnNames":[]},"10":{"slot":10,"name":"Limit","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"100"}}}},"visible":false,"status":{},"columnNames":[]},"11":{"slot":11,"name":"Order By","elements":{},"visible":false,"status":{},"columnNames":[]},"12":{"slot":12,"name":"Sort","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Ascending"}}}},"visible":false,"status":{},"columnNames":[]},"13":{"slot":13,"name":"Filter Conditions","elements":{},"visible":false,"status":{},"columnNames":[]},"14":{"slot":14,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"AND"}}}},"visible":false,"status":{},"columnNames":[]},"20":{"slot":20,"name":"Basic / Advanced","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Advanced"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41425":{"id":41425,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":-480,"y":160,"width":32,"height":32,"inputConnectorIDs":[41441],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[41476],"outputFalseConnectorIDs":[41477],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Target table exists?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"target_table_exists"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Greater than"},"4":{"slot":4,"type":"STRING","value":"0"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41426":{"id":41426,"inputCardinality":"MANY","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1343684451,"x":320,"y":160,"width":32,"height":32,"inputConnectorIDs":[41477,46058],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[48326],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Join paths"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41427":{"id":41427,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1032749985,"x":-160,"y":240,"width":32,"height":32,"inputConnectorIDs":[41446],"outputSuccessConnectorIDs":[41442],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Lookup current high_water_mark from target"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"SQL Query","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${high_water_mark_sql}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Scalar Variable Mapping","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"max_version"},"2":{"slot":2,"type":"STRING","value":"MAX_VERSION"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"max_date_hour"},"2":{"slot":2,"type":"STRING","value":"MAX_DATE_HOUR"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"max_filename"},"2":{"slot":2,"type":"STRING","value":"MAX_FILENAME"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"target_contains_data"},"2":{"slot":2,"type":"STRING","value":"TARGET_CONTAINS_DATA"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"[Environment Default]"}}}},"visible":false,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Table Columns","elements":{},"visible":false,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"[Environment Default]"}}}},"visible":false,"status":{},"columnNames":[]},"10":{"slot":10,"name":"Limit","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"100"}}}},"visible":false,"status":{},"columnNames":[]},"11":{"slot":11,"name":"Order By","elements":{},"visible":false,"status":{},"columnNames":[]},"12":{"slot":12,"name":"Sort","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Ascending"}}}},"visible":false,"status":{},"columnNames":[]},"13":{"slot":13,"name":"Filter Conditions","elements":{},"visible":false,"status":{},"columnNames":[]},"14":{"slot":14,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"AND"}}}},"visible":false,"status":{},"columnNames":[]},"20":{"slot":20,"name":"Basic / Advanced","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Advanced"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41428":{"id":41428,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":-1920,"y":160,"width":32,"height":32,"inputConnectorIDs":[41445],"outputSuccessConnectorIDs":[41447],"outputFailureConnectorIDs":[41481],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Lookup connector specific config"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"from pprint import pprint\n\n# get grid variable containing connector config\nconnector_config = context.getGridVariable('connector_config') or []\nprint('connector_config:')\npprint(connector_config, width=150)\nprint()\n\nconnector = connector.lower().strip()\nprint(f'connector: {connector!r}')\ncontext.updateVariable('connector', connector)\n\n# lookup config for current connector\nfor c, cid, cid_dt, cid_p, cid_s, cid_dbt in connector_config:\n if c == connector:\n print(f'commit_id_name: {cid!r}')\n context.updateVariable('commit_id_name', cid)\n \n print(f'commit_id_data_type: {cid_dt!r}')\n context.updateVariable('commit_id_datatype', cid_dt)\n \n print(f'commit_id_precision: {cid_p!r}')\n context.updateVariable('commit_id_precision', cid_p)\n \n print(f'commit_id_scale: {cid_s!r}')\n context.updateVariable('commit_id_scale', cid_s)\n \n print(f'commit_id_database_type: {cid_dbt!r}')\n context.updateVariable('commit_id_database_type', cid_dbt)\n \n break\n\nif not commit_id_name:\n supported_connectors = ', '.join([c for c, cid in connector_config])\n err_msg = f'This external table is populated with data from the {connector!r} connector, but this job only supports the connectors: {supported_connectors}'\n raise Exception(err_msg)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41429":{"id":41429,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":-1920,"y":320,"width":32,"height":32,"inputConnectorIDs":[41481],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Failure - Unknown connector"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41430":{"id":41430,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":0,"y":240,"width":32,"height":32,"inputConnectorIDs":[41442],"outputSuccessConnectorIDs":[46033],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Print high_water_mark"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"print(f'max_version: {max_version!r}')\nprint(f'max_date_hour: {max_date_hour!r}')\nprint(f'max_filename: {max_filename!r}')\nprint(f'target_contains_data: {target_contains_data!r}')\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41431":{"id":41431,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":-1760,"y":160,"width":32,"height":32,"inputConnectorIDs":[41447],"outputSuccessConnectorIDs":[43104],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Update [Environment Default] values"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"if warehouse == '[Environment Default]':\n print(f'Updating warehouse from [Environment Default] to: {environment_port!r}')\n context.updateVariable('warehouse', environment_port)\n\nif target_database == '[Environment Default]':\n print(f'Updating target_database from [Environment Default] to: {environment_database!r}')\n context.updateVariable('target_database', environment_database)\n\nif target_schema == '[Environment Default]':\n print(f'Updating target_schema from [Environment Default] to: {environment_default_schema!r}')\n context.updateVariable('target_schema', environment_default_schema)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41432":{"id":41432,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":-800,"y":160,"width":32,"height":32,"inputConnectorIDs":[41472],"outputSuccessConnectorIDs":[41444],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Build target_table_exists_sql"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"target_table_exists_sql = \\\nf\"\"\"SELECT \n count(1) AS \"TABLE_EXISTS\"\nFROM \n \"{target_database}\".\"INFORMATION_SCHEMA\".\"TABLES\"\nWHERE \n \"TABLE_SCHEMA\" = '{target_schema}'\n AND \"TABLE_NAME\" = '{target_table}'\"\"\"\n\nprint(f'target_table_exists_sql: \\n{target_table_exists_sql}')\ncontext.updateVariable('target_table_exists_sql', target_table_exists_sql)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41433":{"id":41433,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":-320,"y":240,"width":32,"height":32,"inputConnectorIDs":[41476],"outputSuccessConnectorIDs":[41446],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Build high_water_mark_sql"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"high_water_mark_sql = \\\nf'''WITH \n\"LAST_ROW\" AS (\n SELECT \n \"MTLN_CDC_SRC_VERSION\",\n \"MTLN_CDC_PROCESSED_DATE_HOUR\",\n \"MTLN_CDC_FILENAME\"\n FROM \n \"{target_database}\".\"{target_schema}\".\"{target_table}\"\n ORDER BY\n \"MTLN_CDC_SRC_VERSION\" DESC,\n \"MTLN_CDC_PROCESSED_DATE_HOUR\" DESC,\n \"MTLN_CDC_FILENAME\" DESC\n LIMIT 1\n)\nSELECT \n coalesce(any_value(\"MTLN_CDC_SRC_VERSION\"), 0) AS \"MAX_VERSION\",\n coalesce(any_value(\"MTLN_CDC_PROCESSED_DATE_HOUR\"), '1900-01-01') AS \"MAX_DATE_HOUR\",\n coalesce(any_value(\"MTLN_CDC_FILENAME\"), '0000000000.avro') AS \"MAX_FILENAME\",\n count(1) AS TARGET_CONTAINS_DATA\nFROM \n \"LAST_ROW\"'''\n\nprint(f'high_water_mark_sql: \\n{high_water_mark_sql}')\ncontext.updateVariable('high_water_mark_sql', high_water_mark_sql)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41434":{"id":41434,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":-2080,"y":160,"width":32,"height":32,"inputConnectorIDs":[50232],"outputSuccessConnectorIDs":[41445],"outputFailureConnectorIDs":[41482],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Validate input"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"def validate_input(name: str, description: str, value: str, valid_options: list) -> None:\n \"\"\"\n Standardise input paramater, and check it is in list of valid options\n \"\"\"\n value = value.lower()\n\n\t# check value is valid\n if value not in valid_options:\n err_msg = f\"Invalid value for {description}. Value supplied was '{value!r}', but the valid options are {valid_options}.\"\n raise Exception(err_msg)\n\n print(f'{name}: {value!r}')\n context.updateVariable(name, value)\n\n\nvalidate_input(\n name = 'transformation_type',\n description = 'Transformation Type',\n value = transformation_type,\n valid_options = ['copy table', 'copy table with soft deletes', 'change log']\n)\n\nvalidate_input(\n name = 'schema_drift_action', \n description = 'Schema Drift Action',\n value = schema_drift_action,\n valid_options = ['update target', 'fail job']\n)\n\nif not cloud_storage_url[-1:] == '/':\n cloud_storage_url = cloud_storage_url + '/'\n \nprint(f'cloud_storage_url: {cloud_storage_url!r}')\ncontext.updateVariable('cloud_storage_url', cloud_storage_url)\n\nif trim_all_strings.lower() in ('yes', 'y', 'true', 't', '1'):\n trim_all_strings = 'y'\nelse:\n trim_all_strings = 'n'\n\nprint(f'trim_all_strings: {trim_all_strings!r}')\ncontext.updateVariable('trim_all_strings', trim_all_strings)\n\nif tag_processed_files.lower() in ('yes', 'y', 'true', 't', '1'):\n tag_processed_files = 'y'\nelse:\n tag_processed_files = 'n'\n\nprint(f'tag_processed_files: {tag_processed_files!r}')\ncontext.updateVariable('tag_processed_files', tag_processed_files)\n\nif use_source_schemas.lower() in ('yes', 'y', 'true', 't', '1'):\n use_source_schemas = 'y'\nelse:\n use_source_schemas = 'n'\n\nprint(f'use_source_schemas: {use_source_schemas!r}')\ncontext.updateVariable('use_source_schemas', use_source_schemas)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41435":{"id":41435,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":-1040,"y":160,"width":32,"height":32,"inputConnectorIDs":[43461],"outputSuccessConnectorIDs":[41472],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Build target table name"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"if not target_prefix:\n target_prefix = ''\n\nprint(f'fully_qualify_target_table: {fully_qualify_target_table}')\n \nif fully_qualify_target_table.lower() in ('yes', 'y', 'true', 't', '1'):\n print('Using fully qualified prefix')\n fully_qualified_prefix = f'{source_database}_{source_schema}_'\nelse:\n print('Not using fully qualified prefix')\n fully_qualified_prefix = ''\n \ntarget_table = f'{target_prefix}{fully_qualified_prefix}{source_table}'.strip().upper()\n\nprint()\nprint(f'target_table: {target_table}')\ncontext.updateVariable('target_table', target_table)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41436":{"id":41436,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":-2080,"y":320,"width":32,"height":32,"inputConnectorIDs":[41482],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Failure - Input failed validation"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41437":{"id":41437,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":1600,"y":192,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"target_table_exists","mapTo":"target_table_exists","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Process source metadata version"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync Single Table - 1-2 - Process source metadata version"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"append_metadata"},"2":{"slot":2,"type":"STRING","value":"${append_metadata}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"commit_id_name"},"2":{"slot":2,"type":"STRING","value":"${commit_id_name}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"commit_id_datatype"},"2":{"slot":2,"type":"STRING","value":"${commit_id_datatype}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"commit_id_precision"},"2":{"slot":2,"type":"STRING","value":"${commit_id_precision}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"commit_id_scale"},"2":{"slot":2,"type":"STRING","value":"${commit_id_scale}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"commit_id_database_type"},"2":{"slot":2,"type":"STRING","value":"${commit_id_database_type}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"connector"},"2":{"slot":2,"type":"STRING","value":"${connector}"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"min_date_hour"},"2":{"slot":2,"type":"STRING","value":"${min_date_hour}"}}},"9":{"slot":9,"values":{"1":{"slot":1,"type":"STRING","value":"min_filename"},"2":{"slot":2,"type":"STRING","value":"${min_filename}"}}},"10":{"slot":10,"values":{"1":{"slot":1,"type":"STRING","value":"schema_drift_action"},"2":{"slot":2,"type":"STRING","value":"${schema_drift_action}"}}},"11":{"slot":11,"values":{"1":{"slot":1,"type":"STRING","value":"source_database"},"2":{"slot":2,"type":"STRING","value":"${source_database}"}}},"12":{"slot":12,"values":{"1":{"slot":1,"type":"STRING","value":"source_schema"},"2":{"slot":2,"type":"STRING","value":"${source_schema}"}}},"13":{"slot":13,"values":{"1":{"slot":1,"type":"STRING","value":"source_table"},"2":{"slot":2,"type":"STRING","value":"${source_table}"}}},"14":{"slot":14,"values":{"1":{"slot":1,"type":"STRING","value":"source_version"},"2":{"slot":2,"type":"STRING","value":"${source_version}"}}},"15":{"slot":15,"values":{"1":{"slot":1,"type":"STRING","value":"stage_schema"},"2":{"slot":2,"type":"STRING","value":"${stage_schema}"}}},"16":{"slot":16,"values":{"1":{"slot":1,"type":"STRING","value":"stage_table"},"2":{"slot":2,"type":"STRING","value":"${stage_table}"}}},"17":{"slot":17,"values":{"1":{"slot":1,"type":"STRING","value":"target_database"},"2":{"slot":2,"type":"STRING","value":"${target_database}"}}},"18":{"slot":18,"values":{"1":{"slot":1,"type":"STRING","value":"target_schema"},"2":{"slot":2,"type":"STRING","value":"${target_schema}"}}},"19":{"slot":19,"values":{"1":{"slot":1,"type":"STRING","value":"target_table"},"2":{"slot":2,"type":"STRING","value":"${target_table}"}}},"20":{"slot":20,"values":{"1":{"slot":1,"type":"STRING","value":"target_table_exists"},"2":{"slot":2,"type":"STRING","value":"${target_table_exists}"}}},"21":{"slot":21,"values":{"1":{"slot":1,"type":"STRING","value":"transformation_type"},"2":{"slot":2,"type":"STRING","value":"${transformation_type}"}}},"22":{"slot":22,"values":{"1":{"slot":1,"type":"STRING","value":"warehouse"},"2":{"slot":2,"type":"STRING","value":"${warehouse}"}}},"23":{"slot":23,"values":{"1":{"slot":1,"type":"STRING","value":"logging_enabled"},"2":{"slot":2,"type":"STRING","value":"${logging_enabled}"}}},"24":{"slot":24,"values":{"1":{"slot":1,"type":"STRING","value":"log_table"},"2":{"slot":2,"type":"STRING","value":"${log_table}"}}},"25":{"slot":25,"values":{"1":{"slot":1,"type":"STRING","value":"trim_all_strings"},"2":{"slot":2,"type":"STRING","value":"${trim_all_strings}"}}},"26":{"slot":26,"values":{"1":{"slot":1,"type":"STRING","value":"cloud_storage_url"},"2":{"slot":2,"type":"STRING","value":"${cloud_storage_url}"}}},"27":{"slot":27,"values":{"1":{"slot":1,"type":"STRING","value":"azure_key_vault_url"},"2":{"slot":2,"type":"STRING","value":"${azure_key_vault_url}"}}},"28":{"slot":28,"values":{"1":{"slot":1,"type":"STRING","value":"storage_account_key_secret"},"2":{"slot":2,"type":"STRING","value":"${storage_account_key_secret}"}}},"29":{"slot":29,"values":{"1":{"slot":1,"type":"STRING","value":"tag_processed_files"},"2":{"slot":2,"type":"STRING","value":"${tag_processed_files}"}}},"30":{"slot":30,"values":{"1":{"slot":1,"type":"STRING","value":"bytes_to_decimal_function"},"2":{"slot":2,"type":"STRING","value":"${bytes_to_decimal_function}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"next_files"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"version"},"2":{"slot":2,"type":"GRID","value":"filepath"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"file_list"},"2":{"slot":2,"type":"STRING","value":"grid"}}},"2":{"slot":2,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"primary_key_override"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"source_database"},"2":{"slot":2,"type":"GRID","value":"source_schema"},"3":{"slot":3,"type":"GRID","value":"source_table"},"4":{"slot":4,"type":"GRID","value":"source_column"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"primary_key_override"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync Single Table (Exchange)"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Data Loader.CDC"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"3"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[41483]},"41438":{"id":41438,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"ITERATE","implementationID":-424773870,"x":1600,"y":160,"width":32,"height":16,"inputConnectorIDs":[60480],"outputSuccessConnectorIDs":[46096],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Iterate over source metadata versions"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Grid Variable","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"metadata_versions"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Grid Variable Mapping","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"source_version"},"2":{"slot":2,"type":"STRING","value":"source_version"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"min_date_hour"},"2":{"slot":2,"type":"STRING","value":"min_date_hour"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"min_filename"},"2":{"slot":2,"type":"STRING","value":"min_filename"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Break on Failure","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Yes"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Concurrency","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sequential"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Stop on Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"No"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":false,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Condition","elements":{},"visible":false,"status":{},"columnNames":[]},"10":{"slot":10,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":false,"status":{},"columnNames":[]},"11":{"slot":11,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"999":{"slot":999,"name":"Record Values In Task History","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Yes"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[41483],"inputIterationConnectorIDs":[]},"41439":{"id":41439,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":1760,"y":160,"width":32,"height":32,"inputConnectorIDs":[46096],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Success"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"43103":{"id":43103,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":-1600,"y":160,"width":32,"height":32,"inputConnectorIDs":[43104],"outputSuccessConnectorIDs":[43958],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"bytes_to_decimal_function","mapTo":"bytes_to_decimal_function","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Validate bytes to decimal function"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync Single Table - 1-0 - Validate bytes to decimal function"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"bytes_to_decimal_function"},"2":{"slot":2,"type":"STRING","value":"${bytes_to_decimal_function}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync Single Table (Exchange)"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Data Loader.CDC"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"3"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"43394":{"id":43394,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":1155583855,"x":720,"y":160,"width":32,"height":32,"inputConnectorIDs":[43389],"outputSuccessConnectorIDs":[59921],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"Row Count","mapTo":"new_files_found","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Check for new files in stage table"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Basic / Advanced","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Advanced"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"[Environment Default]"}}}},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Table Columns","elements":{},"visible":false,"status":{},"columnNames":[]},"7":{"slot":7,"name":"SQL Query","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${next_files_sql}"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Grid Variable","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"next_files"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Grid Variable Mapping","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"VERSION"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"DATE_HOUR"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"FILENAME"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"FILEPATH"}}}},"visible":true,"status":{},"columnNames":[]},"10":{"slot":10,"name":"Limit","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"100"}}}},"visible":false,"status":{},"columnNames":[]},"11":{"slot":11,"name":"Order By","elements":{},"visible":false,"status":{},"columnNames":[]},"12":{"slot":12,"name":"Sort","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Ascending"}}}},"visible":false,"status":{},"columnNames":[]},"13":{"slot":13,"name":"Filter Conditions","elements":{},"visible":false,"status":{},"columnNames":[]},"14":{"slot":14,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"AND"}}}},"visible":false,"status":{},"columnNames":[]},"20":{"slot":20,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"[Environment Default]"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"43425":{"id":43425,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":1395699082,"x":-1280,"y":160,"width":32,"height":32,"inputConnectorIDs":[43961],"outputSuccessConnectorIDs":[43461],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Filter primary_key_override for source table"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Target Grid Variable","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"primary_key_override"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Key Columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"source_database"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"source_schema"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"source_table"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Operation","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"In"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Fixed/Grid","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Fixed"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Values Grid","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${source_database}"},"2":{"slot":2,"type":"STRING","value":"${source_schema}"},"3":{"slot":3,"type":"STRING","value":"${source_table}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Source Grid Variable","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Key Column Mapping","elements":{},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"43957":{"id":43957,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":-1434,"y":156,"width":32,"height":32,"inputConnectorIDs":[43958],"outputSuccessConnectorIDs":[43961],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"stage_schema","mapTo":"stage_schema","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"target_schema","mapTo":"target_schema","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Validate schema"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync Single Table - 1-1 - Validate schema"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"use_source_schemas"},"2":{"slot":2,"type":"STRING","value":"${use_source_schemas}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"source_schema"},"2":{"slot":2,"type":"STRING","value":"${source_schema}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"target_database"},"2":{"slot":2,"type":"STRING","value":"${target_database}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"target_schema"},"2":{"slot":2,"type":"STRING","value":"${target_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync Single Table (Exchange)"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Data Loader.CDC"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"3"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"45315":{"id":45315,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":160,"y":400,"width":32,"height":32,"inputConnectorIDs":[46055],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Failure - NULL values detected in high-water mark columns"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"46028":{"id":46028,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":160,"y":240,"width":32,"height":32,"inputConnectorIDs":[46033],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[46055],"outputFalseConnectorIDs":[46058],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Target table contains NULL values?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Advanced"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"target_contains_data"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"1"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"max_version"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"1"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"max_date_hour"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"1900-01-01"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"max_filename"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"0000000000.avro"}}}},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":false,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"target_contains_data == 1 && (max_version == 0 || max_date_hour === '1900-01-01' || max_filename === '0000000000.avro')"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"50171":{"id":50171,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":-2320,"y":160,"width":32,"height":32,"inputConnectorIDs":[50229],"outputSuccessConnectorIDs":[50232],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Print version"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"print('Version: 3.2.0')\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"52726":{"id":52726,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":560,"y":160,"width":32,"height":32,"inputConnectorIDs":[48326],"outputSuccessConnectorIDs":[43389],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Build next_files_sql"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"# Note, when querying Snowflake external tables, even if you only query partition key\n# columns, Snowflake will do a table scan on all the files.\n# To get round this, I'm querying a system function to get a list of all the files in the \n# external table.\nnext_files_sql = \\\nf'''WITH \n\"TABLE_FILES\" AS (\n SELECT \n SPLIT_PART(SPLIT_PART(\"FILE_NAME\", '/', -9), '=', -1) AS \"DATABASE\",\n SPLIT_PART(SPLIT_PART(\"FILE_NAME\", '/', -8), '=', -1) AS \"SCHEMA\",\n SPLIT_PART(SPLIT_PART(\"FILE_NAME\", '/', -7), '=', -1) AS \"TABLE\",\n CAST(SPLIT_PART(SPLIT_PART(\"FILE_NAME\", '/', -6), '=', -1) AS NUMBER(38, 0)) as \"VERSION\",\n TO_TIMESTAMP(SPLIT_PART(SPLIT_PART(\"FILE_NAME\", '/', -5), '=', -1) || SPLIT_PART(SPLIT_PART(\"FILE_NAME\", '/', -4), '=', -1) || SPLIT_PART(SPLIT_PART(\"FILE_NAME\", '/', -3), '=', -1) || SPLIT_PART(SPLIT_PART(\"FILE_NAME\", '/', -2), '=', -1), 'YYYYMMddHH') AS \"DATE_HOUR\",\n SPLIT_PART(\"FILE_NAME\", '/', -1) AS \"FILENAME\",\n \"FILE_NAME\" AS \"FILEPATH\"\n FROM \n table(\"{target_database}\".\"INFORMATION_SCHEMA\".\"EXTERNAL_TABLE_FILES\"(table_name=>'\"{target_database}\".\"{stage_schema}\".\"{stage_table}\"'))\n)\nSELECT \n \"VERSION\",\n \"DATE_HOUR\",\n \"FILENAME\",\n \"FILEPATH\"\nFROM \n \"TABLE_FILES\"\nWHERE\n \"DATABASE\" = '{source_database}'\n AND \"SCHEMA\" = '{source_schema}'\n AND \"TABLE\" = '{source_table}'\n AND (\n (\n \"VERSION\" = '{max_version}'\n AND \"DATE_HOUR\" = '{max_date_hour}'\n \t AND \"FILENAME\" > '{max_filename}'\n )\n OR\n (\n \"VERSION\" = '{max_version}'\n AND \"DATE_HOUR\" > '{max_date_hour}'\n )\n OR\n (\n \"VERSION\" > '{max_version}'\n )\n )\nORDER BY\n \"VERSION\",\n \"DATE_HOUR\",\n \"FILENAME\"'''\n\nprint(f'next_files_sql: \\n{next_files_sql}')\ncontext.updateVariable('next_files_sql', next_files_sql)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"52976":{"id":52976,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":1360,"y":160,"width":32,"height":32,"inputConnectorIDs":[60410],"outputSuccessConnectorIDs":[60480],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Print source metadata versions"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"metadata_versions = context.getGridVariable('metadata_versions') or []\n\nprint('metadata_versions:')\nfor row in metadata_versions:\n print(\"\\t\".join(str(col) for col in row))\ncontext.updateGridVariable('metadata_versions', metadata_versions)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"59945":{"id":59945,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":1040,"y":160,"width":32,"height":32,"inputConnectorIDs":[59959],"outputSuccessConnectorIDs":[60167],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Build get_source_versions_sql"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"# Note, when querying Snowflake external tables, even if you only query partition key\n# columns, Snowflake will do a table scan on all the files.\n# To get round this, I'm querying a system function to get a list of all the files in the \n# external table.\nget_source_versions_sql = \\\nf'''WITH \n\"TABLE_FILES\" AS (\n SELECT \n SPLIT_PART(SPLIT_PART(\"FILE_NAME\", '/', -9), '=', -1) AS \"DATABASE\",\n SPLIT_PART(SPLIT_PART(\"FILE_NAME\", '/', -8), '=', -1) AS \"SCHEMA\",\n SPLIT_PART(SPLIT_PART(\"FILE_NAME\", '/', -7), '=', -1) AS \"TABLE\",\n CAST(SPLIT_PART(SPLIT_PART(\"FILE_NAME\", '/', -6), '=', -1) AS NUMBER(38, 0)) as \"VERSION\",\n TO_TIMESTAMP(SPLIT_PART(SPLIT_PART(\"FILE_NAME\", '/', -5), '=', -1) || SPLIT_PART(SPLIT_PART(\"FILE_NAME\", '/', -4), '=', -1) || SPLIT_PART(SPLIT_PART(\"FILE_NAME\", '/', -3), '=', -1) || SPLIT_PART(SPLIT_PART(\"FILE_NAME\", '/', -2), '=', -1), 'YYYYMMddHH') AS \"DATE_HOUR\",\n SPLIT_PART(\"FILE_NAME\", '/', -1) AS \"FILENAME\"\n FROM \n table(\"{target_database}\".\"INFORMATION_SCHEMA\".\"EXTERNAL_TABLE_FILES\"(table_name=>'\"{target_database}\".\"{stage_schema}\".\"{stage_table}\"'))\n)\nSELECT \n \"VERSION\",\n \"DATE_HOUR\",\n \"FILENAME\",\n ROW_NUMBER() OVER(PARTITION BY \"VERSION\" ORDER BY \"DATE_HOUR\", \"FILENAME\") AS \"ROW_ID\"\nFROM \n \"TABLE_FILES\"\nWHERE\n \"DATABASE\" = '{source_database}'\n AND \"SCHEMA\" = '{source_schema}'\n AND \"TABLE\" = '{source_table}'\n AND (\n (\n \"VERSION\" = '{max_version}'\n AND \"DATE_HOUR\" = '{max_date_hour}'\n \t AND \"FILENAME\" > '{max_filename}'\n )\n OR\n (\n \"VERSION\" = '{max_version}'\n AND \"DATE_HOUR\" > '{max_date_hour}'\n )\n OR\n (\n \"VERSION\" > '{max_version}'\n )\n )\nQUALIFY\n \"ROW_ID\" = 1\nORDER BY\n \"VERSION\",\n \"DATE_HOUR\",\n \"FILENAME\"'''\n\nprint(f'get_source_versions_sql: \\n{get_source_versions_sql}')\ncontext.updateVariable('get_source_versions_sql', get_source_versions_sql)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"60125":{"id":60125,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":1155583855,"x":1200,"y":160,"width":32,"height":32,"inputConnectorIDs":[60167],"outputSuccessConnectorIDs":[60410],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"Row Count","mapTo":"new_files_found","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Get source metadata versions"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Basic / Advanced","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Advanced"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"[Environment Default]"}}}},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Table Columns","elements":{},"visible":false,"status":{},"columnNames":[]},"7":{"slot":7,"name":"SQL Query","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${get_source_versions_sql}"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Grid Variable","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"metadata_versions"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Grid Variable Mapping","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"VERSION"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"DATE_HOUR"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"FILENAME"}}}},"visible":true,"status":{},"columnNames":[]},"10":{"slot":10,"name":"Limit","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"100"}}}},"visible":false,"status":{},"columnNames":[]},"11":{"slot":11,"name":"Order By","elements":{},"visible":false,"status":{},"columnNames":[]},"12":{"slot":12,"name":"Sort","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Ascending"}}}},"visible":false,"status":{},"columnNames":[]},"13":{"slot":13,"name":"Filter Conditions","elements":{},"visible":false,"status":{},"columnNames":[]},"14":{"slot":14,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"AND"}}}},"visible":false,"status":{},"columnNames":[]},"20":{"slot":20,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"[Environment Default]"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"41441":{"id":41441,"sourceID":41424,"targetID":41425},"41442":{"id":41442,"sourceID":41427,"targetID":41430},"41444":{"id":41444,"sourceID":41432,"targetID":41424},"41445":{"id":41445,"sourceID":41434,"targetID":41428},"41446":{"id":41446,"sourceID":41433,"targetID":41427},"41447":{"id":41447,"sourceID":41428,"targetID":41431},"41472":{"id":41472,"sourceID":41435,"targetID":41432},"43104":{"id":43104,"sourceID":41431,"targetID":43103},"43389":{"id":43389,"sourceID":52726,"targetID":43394},"43461":{"id":43461,"sourceID":43425,"targetID":41435},"43958":{"id":43958,"sourceID":43103,"targetID":43957},"43961":{"id":43961,"sourceID":43957,"targetID":43425},"46033":{"id":46033,"sourceID":41430,"targetID":46028},"46096":{"id":46096,"sourceID":41438,"targetID":41439},"50232":{"id":50232,"sourceID":50171,"targetID":41434},"59921":{"id":59921,"sourceID":43394,"targetID":41421},"60167":{"id":60167,"sourceID":59945,"targetID":60125},"60410":{"id":60410,"sourceID":60125,"targetID":52976},"60480":{"id":60480,"sourceID":52976,"targetID":41438}},"failureConnectors":{"41481":{"id":41481,"sourceID":41428,"targetID":41429},"41482":{"id":41482,"sourceID":41434,"targetID":41436}},"unconditionalConnectors":{"48326":{"id":48326,"sourceID":41426,"targetID":52726},"50229":{"id":50229,"sourceID":41423,"targetID":50171}},"trueConnectors":{"41476":{"id":41476,"sourceID":41425,"targetID":41433},"46055":{"id":46055,"sourceID":46028,"targetID":45315},"59959":{"id":59959,"sourceID":41421,"targetID":59945}},"falseConnectors":{"41477":{"id":41477,"sourceID":41425,"targetID":41426},"41478":{"id":41478,"sourceID":41421,"targetID":41422},"46058":{"id":46058,"sourceID":46028,"targetID":41426}},"iterationConnectors":{"41483":{"id":41483,"sourceID":41438,"targetID":41437}},"noteConnectors":{},"canUndo":false,"undoCommand":"","undoCreated":0,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"41413":{"id":41413,"x":-876,"y":4,"width":1268,"height":309,"text":"**Check if target table exists**\n\nCheck if the target table exists. If it does, attempt to read the \ncurrent maximum value for the high-water mark columns.","colour":"e6e63c"},"41414":{"id":41414,"x":-3194,"y":-232,"width":628,"height":145,"text":"**Sync a target table with the source table using CDC event data in an external table**\n\nFirst execution\n- Target table created in Snowflake with primary key\n- All event data in external table used to populate target table with changes\n\nSubsequent execution\n- Lookup last file in target table\n- All event data after this file in external table used to update the target table","colour":"00ce4f"},"41415":{"id":41415,"x":-2154,"y":6,"width":951,"height":229,"text":"**Clean up input parameters**\n\nValidate input parameters\nSet connector specific variables\nReplace any [Environment Default] parameters with the actual value\nValidate __bytes to decimal__function","colour":"e6e63c"},"41416":{"id":41416,"x":-1116,"y":4,"width":151,"height":228,"text":"**Build target table name**\n\nBuild __target_table__ from __target_prefix__, __source_database__, __source_schema__and __source_table__\n","colour":"e6e63c"},"41417":{"id":41417,"x":-3194,"y":6,"width":628,"height":946,"text":"**Change log**\n\n1.0.0 -----------------------------------------------------------------------------------------------------------\n- General availability\n- Support for PostgreSQL, Oracle and SQL Server sources\n- Support for S3 and Azure Blob Storage locations\n- Support for Copy Table, Copy Table With Soft Deletes and Change Log transformation types\n- Support for schema drift\n\n1.1.0 -----------------------------------------------------------------------------------------------------------\n- Support for Google Cloud Storage locations\n\n2.0.0 -----------------------------------------------------------------------------------------------------------\n- Use filename as the high-water mark, and end the job with success if there are no new files to process\n- Support for MySQL sources\n- Transformation Type will be updated to Change Log if no primary key is detected in the change events\n\n2.0.1 -----------------------------------------------------------------------------------------------------------\n- Fixed ImportError in Python scripts when using Python 3.7\n\n2.1.0 -----------------------------------------------------------------------------------------------------------\n- Adding row count logging from transformation jobs\n\n2.2.0 -----------------------------------------------------------------------------------------------------------\n- Support for DB2 for IBM i sources\n\n2.3.0 -----------------------------------------------------------------------------------------------------------\n- Columns created as Date, Time and Timestamp when the pipeline is set to __time.precision.mode = connect__\n\n2.3.1 -----------------------------------------------------------------------------------------------------------\n- Detect NULL values in high-water mark columns, and raise error\n\n2.3.2 -----------------------------------------------------------------------------------------------------------\n- Add unique alias to external table columns to ensure they don't clash with source table columns\n\n2.4.0 -----------------------------------------------------------------------------------------------------------\n- Optionally add tag to each Avro file after the data has been loaded\n\n2.5.0 -----------------------------------------------------------------------------------------------------------\n- Added parameter for User Defined Function to convert byte arrays to decimals\n\n3.0.0 -----------------------------------------------------------------------------------------------------------\n- For postgresql, switched to using metadata.sequence as source for MTLN_CDC_SEQUENCE_NUMBER, changing data type from NUMBER to VARCHAR. Existing pipelines will be converted to the new format.\n- For oracle, switched to using metadata.rs_id and metadata.ssn as source for MTLN_CDC_SEQUENCE_NUMBER, changing data type from NUMBER to VARCHAR. Existing pipelines must be cleared down and re-snapshot.\n\n3.0.1 -----------------------------------------------------------------------------------------------------------\n- Prevent schema drift code from updating MTLN_CDC_SEQUENCE_NUMBER in existing Oracle pipelines from NUMBER to VARCHAR\n\n3.1.0 -----------------------------------------------------------------------------------------------------------\n- Added optional Primary Key Override parameter so that a primary key can be used for Copy Table transformations when the source table does not have the primary key defined in DDL\n- Added optional Use Source Schemas parameter to create target tables in schemas with the same name as the source. The Target Schema is still used to contain the external tables\n\n3.1.1 -----------------------------------------------------------------------------------------------------------\n- Use SQL to get list of source versions, rather than Python, to remove risk of unsorted list in Python script\n\n3.1.2 -----------------------------------------------------------------------------------------------------------\n- Increased Python Script timeout to 600 seconds on __Tag files - Azure__\n- Added check for a hierarchical namespace before attempting to tag files in Azure\n\n3.2.0 -----------------------------------------------------------------------------------------------------------\n- Updated version of Python in UDF to 3.11\n","colour":"00ce4f"},"41418":{"id":41418,"x":1525,"y":4,"width":148,"height":230,"text":"**Iterate over source metadata versions**\n\nIterate over version list, processing any files for each version in turn.","colour":"e6e63c"},"48349":{"id":48349,"x":487,"y":6,"width":946,"height":228,"text":"**Check for new files to process in the external table**\n\nUse the previous high-water mark values to look for new files in the external table.\n\nIf files are found, build a list of all source versions to iterate over.","colour":"e6e63c"},"50172":{"id":50172,"x":-2394,"y":5,"width":152,"height":229,"text":"**Print version**\n\nPrint shared job version to the task history, to help support cases","colour":"e6e63c"}},"variables":{"connector":{"definition":{"name":"connector","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"log_table":{"definition":{"name":"log_table","type":"TEXT","scope":"TASKBATCH","description":"The name of the table in Snowflake containing the logged events.","visibility":"PUBLIC"},"value":null},"warehouse":{"definition":{"name":"warehouse","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"max_version":{"definition":{"name":"max_version","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"0"},"stage_table":{"definition":{"name":"stage_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"max_filename":{"definition":{"name":"max_filename","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"0000000000.avro"},"min_filename":{"definition":{"name":"min_filename","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"source_table":{"definition":{"name":"source_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"stage_schema":{"definition":{"name":"stage_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"target_table":{"definition":{"name":"target_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"max_date_hour":{"definition":{"name":"max_date_hour","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"1900-01-01"},"min_date_hour":{"definition":{"name":"min_date_hour","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"source_schema":{"definition":{"name":"source_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_prefix":{"definition":{"name":"target_prefix","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_schema":{"definition":{"name":"target_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"commit_id_name":{"definition":{"name":"commit_id_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"next_files_sql":{"definition":{"name":"next_files_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"source_version":{"definition":{"name":"source_version","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"1"},"append_metadata":{"definition":{"name":"append_metadata","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"Y"},"commit_id_scale":{"definition":{"name":"commit_id_scale","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"logging_enabled":{"definition":{"name":"logging_enabled","type":"TEXT","scope":"TASKBATCH","description":"Enable logging of row counts from transformation jobs. Options are:\n\nY: Log row counts\nN: Do not log row counts\n\nNote: Logged row counts may not match the number of rows processed by the CDC agent. For the Copy Table and Copy Table with Soft Deletes transformation types, if a given record has more than one change event in a job execution, only the final change event will be processed. The \"after\" object of the final change event contains the net changes from all the previous events.","visibility":"PUBLIC"},"value":null},"new_files_found":{"definition":{"name":"new_files_found","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"0"},"source_database":{"definition":{"name":"source_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_database":{"definition":{"name":"target_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"trim_all_strings":{"definition":{"name":"trim_all_strings","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"Default"},"cloud_storage_url":{"definition":{"name":"cloud_storage_url","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"commit_id_datatype":{"definition":{"name":"commit_id_datatype","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"use_source_schemas":{"definition":{"name":"use_source_schemas","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"N"},"azure_key_vault_url":{"definition":{"name":"azure_key_vault_url","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"commit_id_precision":{"definition":{"name":"commit_id_precision","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"high_water_mark_sql":{"definition":{"name":"high_water_mark_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"schema_drift_action":{"definition":{"name":"schema_drift_action","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"Update Target"},"tag_processed_files":{"definition":{"name":"tag_processed_files","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"N"},"target_table_exists":{"definition":{"name":"target_table_exists","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"0"},"transformation_type":{"definition":{"name":"transformation_type","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"Copy Table"},"target_contains_data":{"definition":{"name":"target_contains_data","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"check_for_versions_sql":{"definition":{"name":"check_for_versions_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"commit_id_database_type":{"definition":{"name":"commit_id_database_type","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"get_source_versions_sql":{"definition":{"name":"get_source_versions_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"target_table_exists_sql":{"definition":{"name":"target_table_exists_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"bytes_to_decimal_function":{"definition":{"name":"bytes_to_decimal_function","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"BYTES_TO_DECIMAL"},"fully_qualify_target_table":{"definition":{"name":"fully_qualify_target_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"Y"},"storage_account_key_secret":{"definition":{"name":"storage_account_key_secret","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""}},"grids":{"next_files":{"definition":{"name":"next_files","scope":"TASKBATCH","definitions":[{"name":"version","type":"DECIMAL"},{"name":"date_hour","type":"TEXT"},{"name":"filename","type":"TEXT"},{"name":"filepath","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[]},"connector_config":{"definition":{"name":"connector_config","scope":"TASKBATCH","definitions":[{"name":"connector","type":"TEXT"},{"name":"commit_id_name","type":"TEXT"},{"name":"commit_id_data_type","type":"TEXT"},{"name":"commit_id_precision","type":"TEXT"},{"name":"commit_id_scale","type":"TEXT"},{"name":"commit_id_database_type","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[{"values":["postgresql","sequence","VARCHAR","16777216","0","VARCHAR(16777216)"]},{"values":["sqlserver","change_lsn","VARCHAR","16777216","0","VARCHAR(16777216)"]},{"values":["oracle","rs_id","VARCHAR","16777216","0","VARCHAR(16777216)"]},{"values":["mysql","file_pos_row","VARCHAR","16777216","0","VARCHAR(16777216)"]},{"values":["db2_ibm_i","event_serial_number","NUMBER","38","0","NUMBER(38, 0)"]}]},"metadata_versions":{"definition":{"name":"metadata_versions","scope":"TASKBATCH","definitions":[{"name":"source_version","type":"DECIMAL"},{"name":"min_date_hour","type":"TEXT"},{"name":"min_filename","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[]},"primary_key_override":{"definition":{"name":"primary_key_override","scope":"TASKBATCH","definitions":[{"name":"source_database","type":"TEXT"},{"name":"source_schema","type":"TEXT"},{"name":"source_table","type":"TEXT"},{"name":"source_column","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]}}},"Sync Single Table - 1-2-5 - Transform and Load":{"id":-2,"revision":1,"created":1728917529394,"timestamp":1728917529394,"components":{"33617":{"id":33617,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":960,"y":960,"width":32,"height":32,"inputConnectorIDs":[38726],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log row counts for transform and load"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Logging - 1 - Log Metrics"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"database"},"2":{"slot":2,"type":"STRING","value":"${target_database}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"schema"},"2":{"slot":2,"type":"STRING","value":"${target_schema}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"warehouse"},"2":{"slot":2,"type":"STRING","value":"${warehouse}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"log_namespace"},"2":{"slot":2,"type":"STRING","value":"${target_table}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"logging_enabled"},"2":{"slot":2,"type":"STRING","value":"${logging_enabled}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"log_table"},"2":{"slot":2,"type":"STRING","value":"${log_table}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"log_level"},"2":{"slot":2,"type":"STRING","value":"INFO"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"exported_from_name"},"2":{"slot":2,"type":"STRING","value":"${exported_from_name}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Inserted Row Count"},"2":{"slot":2,"type":"STRING","value":"${logged_rows_inserted}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"Updated Row Count"},"2":{"slot":2,"type":"STRING","value":"${logged_rows_updated}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"Deleted Row Count"},"2":{"slot":2,"type":"STRING","value":"${logged_rows_deleted}"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"logging_message"},"2":{"slot":2,"type":"STRING","value":"values"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync Single Table (Exchange)"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Data Loader.CDC"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"3"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"33856":{"id":33856,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":560,"y":640,"width":32,"height":32,"inputConnectorIDs":[38712],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log row counts for transform and append"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Logging - 1 - Log Metrics"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"database"},"2":{"slot":2,"type":"STRING","value":"${target_database}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"schema"},"2":{"slot":2,"type":"STRING","value":"${target_schema}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"warehouse"},"2":{"slot":2,"type":"STRING","value":"${warehouse}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"log_namespace"},"2":{"slot":2,"type":"STRING","value":"${target_table}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"logging_enabled"},"2":{"slot":2,"type":"STRING","value":"${logging_enabled}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"log_table"},"2":{"slot":2,"type":"STRING","value":"${log_table}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"log_level"},"2":{"slot":2,"type":"STRING","value":"INFO"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"exported_from_name"},"2":{"slot":2,"type":"STRING","value":"${exported_from_name}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Inserted Row Count"},"2":{"slot":2,"type":"STRING","value":"${logged_rows_inserted}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"Updated Row Count"},"2":{"slot":2,"type":"STRING","value":"${logged_rows_updated}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"Deleted Row Count"},"2":{"slot":2,"type":"STRING","value":"${logged_rows_deleted}"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"logging_message"},"2":{"slot":2,"type":"STRING","value":"values"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync Single Table (Exchange)"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Data Loader.CDC"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"3"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"33886":{"id":33886,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":1600,"y":1360,"width":32,"height":32,"inputConnectorIDs":[38679],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log row counts for transform and load soft deletes"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Logging - 1 - Log Metrics"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"database"},"2":{"slot":2,"type":"STRING","value":"${target_database}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"schema"},"2":{"slot":2,"type":"STRING","value":"${target_schema}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"warehouse"},"2":{"slot":2,"type":"STRING","value":"${warehouse}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"log_namespace"},"2":{"slot":2,"type":"STRING","value":"${target_table}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"logging_enabled"},"2":{"slot":2,"type":"STRING","value":"${logging_enabled}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"log_table"},"2":{"slot":2,"type":"STRING","value":"${log_table}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"log_level"},"2":{"slot":2,"type":"STRING","value":"INFO"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"exported_from_name"},"2":{"slot":2,"type":"STRING","value":"${exported_from_name}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Inserted Row Count"},"2":{"slot":2,"type":"STRING","value":"${logged_rows_inserted}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"Updated Row Count"},"2":{"slot":2,"type":"STRING","value":"${logged_rows_updated}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"Deleted Row Count"},"2":{"slot":2,"type":"STRING","value":"${logged_rows_deleted}"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"logging_message"},"2":{"slot":2,"type":"STRING","value":"values"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync Single Table (Exchange)"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Data Loader.CDC"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"3"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"38609":{"id":38609,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":1440,"y":1520,"width":32,"height":32,"inputConnectorIDs":[38669],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log row counts for transform and load without deletes"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Logging - 1 - Log Metrics"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"database"},"2":{"slot":2,"type":"STRING","value":"${target_database}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"schema"},"2":{"slot":2,"type":"STRING","value":"${target_schema}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"warehouse"},"2":{"slot":2,"type":"STRING","value":"${warehouse}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"log_namespace"},"2":{"slot":2,"type":"STRING","value":"${target_table}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"logging_enabled"},"2":{"slot":2,"type":"STRING","value":"${logging_enabled}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"log_table"},"2":{"slot":2,"type":"STRING","value":"${log_table}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"log_level"},"2":{"slot":2,"type":"STRING","value":"INFO"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"exported_from_name"},"2":{"slot":2,"type":"STRING","value":"${exported_from_name}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Inserted Row Count"},"2":{"slot":2,"type":"STRING","value":"${logged_rows_inserted}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"Updated Row Count"},"2":{"slot":2,"type":"STRING","value":"${logged_rows_updated}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"Deleted Row Count"},"2":{"slot":2,"type":"STRING","value":"${logged_rows_deleted}"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"logging_message"},"2":{"slot":2,"type":"STRING","value":"values"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync Single Table (Exchange)"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Data Loader.CDC"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"3"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41527":{"id":41527,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":400,"y":960,"width":32,"height":32,"inputConnectorIDs":[41561],"outputSuccessConnectorIDs":[41555],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Build join_output grid var"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"target_metadata = context.getGridVariable('target_metadata') or []\nprimary_key = context.getGridVariable('primary_key') or []\n\n# if we aren't adding full metadata, target_metadata may not contain MTLN_CDC_LAST_CHANGE_TYPE\n# we need this column in the join_output, so append it now\ntarget_columns = [column for column, *_ in target_metadata]\nif 'MTLN_CDC_LAST_CHANGE_TYPE' not in target_columns:\n target_columns.append('MTLN_CDC_LAST_CHANGE_TYPE')\n\n# add target columns into join_output\njoin_output = [[f'SOURCE.{column}', column] for column in target_columns]\n\n# add one column from primary key in target table into join_output\n# Note: it doesn't matter which column as we be using this column to check for \n# null rows in target table after left join on full primary key\njoin_output.append([f'TARGET.{primary_key[0][0]}', 'MTLN_TARGET_KEY'])\n\n# add current sequence number from target table\n# Note: we will be using this column to compare against the stage table to\n# filter out old change events\njoin_output.append(['TARGET.MTLN_CDC_SEQUENCE_NUMBER', 'MTLN_TARGET_MTLN_CDC_SEQUENCE_NUMBER'])\n\nprint('join_output:')\nfor row in join_output:\n print(\"\\t\".join(str(col) for col in row))\ncontext.updateGridVariable('join_output', join_output)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41529":{"id":41529,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-193159521,"x":1280,"y":1360,"width":32,"height":32,"inputConnectorIDs":[41563],"outputSuccessConnectorIDs":[38666],"outputFailureConnectorIDs":[44462],"outputUnconditionalConnectorIDs":[38669],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"logged_rows_deleted","mapTo":"logged_rows_deleted","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"Component","mapTo":"exported_from_name","validationStatus":null,"validationMessage":null},"3":{"slot":3,"fromId":null,"fromName":"logged_rows_inserted","mapTo":"logged_rows_inserted","validationStatus":null,"validationMessage":null},"4":{"slot":4,"fromId":null,"fromName":"logged_rows_updated","mapTo":"logged_rows_updated","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Transform and load without deletes"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync Single Table - 1-2-5-3 - Transform and load without deletes"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"source_database"},"2":{"slot":2,"type":"STRING","value":"${source_database}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"source_schema"},"2":{"slot":2,"type":"STRING","value":"${source_schema}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"source_table"},"2":{"slot":2,"type":"STRING","value":"${source_table}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"source_version"},"2":{"slot":2,"type":"STRING","value":"${source_version}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"warehouse"},"2":{"slot":2,"type":"STRING","value":"${warehouse}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"target_database"},"2":{"slot":2,"type":"STRING","value":"${target_database}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"stage_schema"},"2":{"slot":2,"type":"STRING","value":"${stage_schema}"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"stage_table"},"2":{"slot":2,"type":"STRING","value":"${stage_table}"}}},"9":{"slot":9,"values":{"1":{"slot":1,"type":"STRING","value":"target_schema"},"2":{"slot":2,"type":"STRING","value":"${target_schema}"}}},"10":{"slot":10,"values":{"1":{"slot":1,"type":"STRING","value":"target_table"},"2":{"slot":2,"type":"STRING","value":"${target_table}"}}},"11":{"slot":11,"values":{"1":{"slot":1,"type":"STRING","value":"min_date_hour"},"2":{"slot":2,"type":"STRING","value":"${min_date_hour}"}}},"12":{"slot":12,"values":{"1":{"slot":1,"type":"STRING","value":"min_filename"},"2":{"slot":2,"type":"STRING","value":"${min_filename}"}}},"13":{"slot":13,"values":{"1":{"slot":1,"type":"STRING","value":"join_expression"},"2":{"slot":2,"type":"STRING","value":"${join_expression}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"calculated_columns"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"calculation"},"2":{"slot":2,"type":"GRID","value":"alias"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"calculated_columns"},"2":{"slot":2,"type":"STRING","value":"grid"}}},"2":{"slot":2,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"join_output"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"input_column"},"2":{"slot":2,"type":"GRID","value":"output_column"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"join_output"},"2":{"slot":2,"type":"STRING","value":"grid"}}},"3":{"slot":3,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"primary_key"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"key_column"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"primary_key"},"2":{"slot":2,"type":"STRING","value":"grid"}}},"4":{"slot":4,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"target_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"column_name"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"target_metadata"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync Single Table (Exchange)"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Data Loader.CDC"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"3"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41530":{"id":41530,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":720,"y":1360,"width":32,"height":32,"inputConnectorIDs":[41559],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[41558],"outputFalseConnectorIDs":[41560],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Append full CDC metadata to target_metadata?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Advanced"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"append_metadata"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"Y"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"append_metadata"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"y"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"append_metadata"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"Yes"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"append_metadata"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"YES"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"append_metadata"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"yes"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"append_metadata"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"True"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"append_metadata"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"true"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"append_metadata"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"TRUE"}}}},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Or"}}}},"visible":false,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"['yes', 'y', 'true', 't', '1'].indexOf(append_metadata.toLowerCase()) !== -1"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41531":{"id":41531,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-369209785,"x":880,"y":1280,"width":32,"height":32,"inputConnectorIDs":[41558],"outputSuccessConnectorIDs":[41552],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Append full CDC metadata to delete_mapping"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Target Grid Variable","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"delete_mapping"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Fixed/Grid","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Fixed"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Values Grid","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_SRC_VERSION"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_LAST_CHANGE_TYPE"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_LAST_COMMIT_TIMESTAMP"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_PROCESSED_DATE_HOUR"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_FILENAME"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_FILEPATH"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_SEQUENCE_NUMBER"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_LOAD_BATCH_ID"}}},"9":{"slot":9,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_LOAD_TIMESTAMP"}}},"10":{"slot":10,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_DELETED"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Source Grid Variable","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Column Mapping","elements":{},"visible":false,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Append/Prepend","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Append"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41532":{"id":41532,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-369209785,"x":880,"y":1440,"width":32,"height":32,"inputConnectorIDs":[41560],"outputSuccessConnectorIDs":[41548],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Append minimal CDC metadata to delete_mapping"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Target Grid Variable","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"delete_mapping"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Fixed/Grid","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Fixed"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Values Grid","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_SRC_VERSION"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_PROCESSED_DATE_HOUR"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_FILENAME"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_SEQUENCE_NUMBER"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_DELETED"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Source Grid Variable","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Column Mapping","elements":{},"visible":false,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Append/Prepend","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Append"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41533":{"id":41533,"inputCardinality":"MANY","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1343684451,"x":1040,"y":1360,"width":32,"height":32,"inputConnectorIDs":[41552,41548],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[41563],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Join paths"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41534":{"id":41534,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-193159521,"x":800,"y":960,"width":32,"height":32,"inputConnectorIDs":[41556],"outputSuccessConnectorIDs":[38721],"outputFailureConnectorIDs":[44447],"outputUnconditionalConnectorIDs":[38726],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"logged_rows_deleted","mapTo":"logged_rows_deleted","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"Component","mapTo":"exported_from_name","validationStatus":null,"validationMessage":null},"3":{"slot":3,"fromId":null,"fromName":"logged_rows_updated","mapTo":"logged_rows_updated","validationStatus":null,"validationMessage":null},"4":{"slot":4,"fromId":null,"fromName":"logged_rows_inserted","mapTo":"logged_rows_inserted","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Transform and load"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync Single Table - 1-2-5-2 - Transform and load"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"source_database"},"2":{"slot":2,"type":"STRING","value":"${source_database}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"source_schema"},"2":{"slot":2,"type":"STRING","value":"${source_schema}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"source_table"},"2":{"slot":2,"type":"STRING","value":"${source_table}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"source_version"},"2":{"slot":2,"type":"STRING","value":"${source_version}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"warehouse"},"2":{"slot":2,"type":"STRING","value":"${warehouse}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"target_database"},"2":{"slot":2,"type":"STRING","value":"${target_database}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"stage_schema"},"2":{"slot":2,"type":"STRING","value":"${stage_schema}"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"stage_table"},"2":{"slot":2,"type":"STRING","value":"${stage_table}"}}},"9":{"slot":9,"values":{"1":{"slot":1,"type":"STRING","value":"target_schema"},"2":{"slot":2,"type":"STRING","value":"${target_schema}"}}},"10":{"slot":10,"values":{"1":{"slot":1,"type":"STRING","value":"target_table"},"2":{"slot":2,"type":"STRING","value":"${target_table}"}}},"11":{"slot":11,"values":{"1":{"slot":1,"type":"STRING","value":"min_date_hour"},"2":{"slot":2,"type":"STRING","value":"${min_date_hour}"}}},"12":{"slot":12,"values":{"1":{"slot":1,"type":"STRING","value":"min_filename"},"2":{"slot":2,"type":"STRING","value":"${min_filename}"}}},"13":{"slot":13,"values":{"1":{"slot":1,"type":"STRING","value":"join_expression"},"2":{"slot":2,"type":"STRING","value":"${join_expression}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"calculated_columns"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"calculation"},"2":{"slot":2,"type":"GRID","value":"alias"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"calculated_columns"},"2":{"slot":2,"type":"STRING","value":"grid"}}},"2":{"slot":2,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"join_output"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"input_column"},"2":{"slot":2,"type":"GRID","value":"output_column"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"join_output"},"2":{"slot":2,"type":"STRING","value":"grid"}}},"3":{"slot":3,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"primary_key"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"key_column"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"primary_key"},"2":{"slot":2,"type":"STRING","value":"grid"}}},"4":{"slot":4,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"target_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"column_name"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"target_metadata"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync Single Table (Exchange)"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Data Loader.CDC"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"3"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41535":{"id":41535,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":960,"y":880,"width":32,"height":32,"inputConnectorIDs":[38721],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Success - Changes applied"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41537":{"id":41537,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":640,"y":960,"width":32,"height":32,"inputConnectorIDs":[41555],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[41556],"outputFalseConnectorIDs":[41559],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Transformation is Copy Table?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"transformation_type"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"copy table"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Or"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"['yes', 'y', 'true', 't', '1'].indexOf(soft_deletes.toLowerCase()) !== -1"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41538":{"id":41538,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-193159521,"x":1440,"y":1360,"width":32,"height":32,"inputConnectorIDs":[38666],"outputSuccessConnectorIDs":[38688],"outputFailureConnectorIDs":[44412],"outputUnconditionalConnectorIDs":[38679],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"logged_rows_deleted","mapTo":"logged_rows_deleted","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"Component","mapTo":"exported_from_name","validationStatus":null,"validationMessage":null},"3":{"slot":3,"fromId":null,"fromName":"logged_rows_inserted","mapTo":"logged_rows_inserted","validationStatus":null,"validationMessage":null},"4":{"slot":4,"fromId":null,"fromName":"logged_rows_updated","mapTo":"logged_rows_updated","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Transform and load soft deletes"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync Single Table - 1-2-5-4 - Transform and load soft deletes"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"source_database"},"2":{"slot":2,"type":"STRING","value":"${source_database}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"source_schema"},"2":{"slot":2,"type":"STRING","value":"${source_schema}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"source_table"},"2":{"slot":2,"type":"STRING","value":"${source_table}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"source_version"},"2":{"slot":2,"type":"STRING","value":"${source_version}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"warehouse"},"2":{"slot":2,"type":"STRING","value":"${warehouse}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"target_database"},"2":{"slot":2,"type":"STRING","value":"${target_database}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"stage_schema"},"2":{"slot":2,"type":"STRING","value":"${stage_schema}"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"stage_table"},"2":{"slot":2,"type":"STRING","value":"${stage_table}"}}},"9":{"slot":9,"values":{"1":{"slot":1,"type":"STRING","value":"target_schema"},"2":{"slot":2,"type":"STRING","value":"${target_schema}"}}},"10":{"slot":10,"values":{"1":{"slot":1,"type":"STRING","value":"target_table"},"2":{"slot":2,"type":"STRING","value":"${target_table}"}}},"11":{"slot":11,"values":{"1":{"slot":1,"type":"STRING","value":"min_date_hour"},"2":{"slot":2,"type":"STRING","value":"${min_date_hour}"}}},"12":{"slot":12,"values":{"1":{"slot":1,"type":"STRING","value":"min_filename"},"2":{"slot":2,"type":"STRING","value":"${min_filename}"}}},"13":{"slot":13,"values":{"1":{"slot":1,"type":"STRING","value":"join_expression"},"2":{"slot":2,"type":"STRING","value":"${join_expression}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"calculated_columns"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"calculation"},"2":{"slot":2,"type":"GRID","value":"alias"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"calculated_columns"},"2":{"slot":2,"type":"STRING","value":"grid"}}},"2":{"slot":2,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"delete_mapping"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"column_name"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"delete_mapping"},"2":{"slot":2,"type":"STRING","value":"grid"}}},"3":{"slot":3,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"join_output"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"input_column"},"2":{"slot":2,"type":"GRID","value":"output_column"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"join_output"},"2":{"slot":2,"type":"STRING","value":"grid"}}},"4":{"slot":4,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"primary_key"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"key_column"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"primary_key"},"2":{"slot":2,"type":"STRING","value":"grid"}}},"5":{"slot":5,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"target_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"column_name"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"target_metadata"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync Single Table (Exchange)"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Data Loader.CDC"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"3"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41539":{"id":41539,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":1600,"y":1280,"width":32,"height":32,"inputConnectorIDs":[38688],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Success - Changes applied with soft deletes"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41540":{"id":41540,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":240,"y":640,"width":32,"height":32,"inputConnectorIDs":[44393],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[49021],"outputFalseConnectorIDs":[41561],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Transformation is Change Log?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"transformation_type"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"change log"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Or"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41541":{"id":41541,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-193159521,"x":400,"y":640,"width":32,"height":32,"inputConnectorIDs":[49021],"outputSuccessConnectorIDs":[38705],"outputFailureConnectorIDs":[44428],"outputUnconditionalConnectorIDs":[38712],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"logged_rows_deleted","mapTo":"logged_rows_deleted","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"Component","mapTo":"exported_from_name","validationStatus":null,"validationMessage":null},"3":{"slot":3,"fromId":null,"fromName":"logged_rows_inserted","mapTo":"logged_rows_inserted","validationStatus":null,"validationMessage":null},"4":{"slot":4,"fromId":null,"fromName":"logged_rows_updated","mapTo":"logged_rows_updated","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Transform and append"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync Single Table - 1-2-5-1 - Transform and append"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"source_database"},"2":{"slot":2,"type":"STRING","value":"${source_database}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"source_schema"},"2":{"slot":2,"type":"STRING","value":"${source_schema}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"source_table"},"2":{"slot":2,"type":"STRING","value":"${source_table}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"source_version"},"2":{"slot":2,"type":"STRING","value":"${source_version}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"warehouse"},"2":{"slot":2,"type":"STRING","value":"${warehouse}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"target_database"},"2":{"slot":2,"type":"STRING","value":"${target_database}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"stage_schema"},"2":{"slot":2,"type":"STRING","value":"${stage_schema}"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"stage_table"},"2":{"slot":2,"type":"STRING","value":"${stage_table}"}}},"9":{"slot":9,"values":{"1":{"slot":1,"type":"STRING","value":"target_schema"},"2":{"slot":2,"type":"STRING","value":"${target_schema}"}}},"10":{"slot":10,"values":{"1":{"slot":1,"type":"STRING","value":"target_table"},"2":{"slot":2,"type":"STRING","value":"${target_table}"}}},"11":{"slot":11,"values":{"1":{"slot":1,"type":"STRING","value":"min_date_hour"},"2":{"slot":2,"type":"STRING","value":"${min_date_hour}"}}},"12":{"slot":12,"values":{"1":{"slot":1,"type":"STRING","value":"min_filename"},"2":{"slot":2,"type":"STRING","value":"${min_filename}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"calculated_columns"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"calculation"},"2":{"slot":2,"type":"GRID","value":"alias"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"calculated_columns"},"2":{"slot":2,"type":"STRING","value":"grid"}}},"2":{"slot":2,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"target_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"column_name"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"target_metadata"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync Single Table (Exchange)"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Data Loader.CDC"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"3"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41542":{"id":41542,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":560,"y":560,"width":32,"height":32,"inputConnectorIDs":[38705],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Success - Changes appended"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41543":{"id":41543,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":-320,"y":640,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[44234],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44233":{"id":44233,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":-80,"y":640,"width":32,"height":32,"inputConnectorIDs":[44234],"outputSuccessConnectorIDs":[44393],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Build calculated_columns"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"def qualified_name(column_name, property_name = None):\n \"\"\"Return a fully qualifed column name with an optional property name.\"\"\"\n if property_name:\n return f'\"{column_name}\":\"{property_name}\"'\n else:\n return f'\"{column_name}\"'\n \n\nnested_metadata = context.getGridVariable('nested_metadata') or []\n\n# Append all the nested columns to calculated_columns\ncalculated_columns = []\nfor prop, alias, format_string, *_ in nested_metadata:\n column = qualified_name('MTLN_CDC_EXT_PAYLOAD', prop)\n if format_string:\n column = format_string.format(col=column)\n calculated_columns.append([column, alias])\n\n# In MySQL, we construct the sequence number from the bin file, row and pos fields\nif connector == 'mysql':\n sequence_number_calc = (\n 'CONCAT_WS('\n \"':', \"\n f\"TO_CHAR(SPLIT_PART({qualified_name('MTLN_CDC_EXT_METADATA', 'file')}, '.', -1)::NUMBER(38, 0), '0000000x'), \"\n f\"TO_CHAR({qualified_name('MTLN_CDC_EXT_METADATA', 'pos')}::NUMBER(38, 0), '000000000000000x'), \"\n f\"TO_CHAR({qualified_name('MTLN_CDC_EXT_METADATA', 'row')}::NUMBER(38, 0), '0000000x')\"\n ')'\n )\n \n# In SQL Server, the change_lsn is null for the initial snapshot, so coalesce with commit_lsn\nelif connector == 'sqlserver':\n sequence_number_calc = (\n 'COALESCE('\n f'{qualified_name(\"MTLN_CDC_EXT_METADATA\", \"change_lsn\")}::VARCHAR(16777216), '\n f'{qualified_name(\"MTLN_CDC_EXT_METADATA\", \"commit_lsn\")}::VARCHAR(16777216)'\n ')'\n )\n \n# In Oracle, we need to sort by rs_id then ssn\nelif connector == 'oracle':\n sequence_number_calc = (\n 'CONCAT_WS('\n \"'.', \"\n f\"COALESCE({qualified_name('MTLN_CDC_EXT_METADATA', 'rs_id')}::VARCHAR, '0x000000.00000000.0000'), \"\n f\"TO_CHAR(COALESCE({qualified_name('MTLN_CDC_EXT_METADATA', 'ssn')}::NUMBER(38, 0), 0), '000000000000000x') \"\n ')'\n )\n\n# In Postgres, check which column we are using for the sequence number\nelif connector == 'postgresql' and commit_id_name == 'sequence':\n sequence_number_calc = (\n 'CONCAT_WS('\n \"':', \"\n f\"TO_CHAR(COALESCE(PARSE_JSON({qualified_name('MTLN_CDC_EXT_METADATA', 'sequence')})[0]::NUMBER(38, 0), 0), '000000000000000x'), \"\n f\"TO_CHAR(COALESCE(PARSE_JSON({qualified_name('MTLN_CDC_EXT_METADATA', 'sequence')})[1]::NUMBER(38, 0), 0), '000000000000000x') \"\n ')'\n )\n\nelse:\n sequence_number_calc = qualified_name('MTLN_CDC_EXT_METADATA', commit_id_name)\n \n# Add all metadata columns\n# Note: any columns not required in the target table will be ignored by the write\n# component in the transformation job\ncalculated_columns.extend([\n [qualified_name('MTLN_CDC_EXT_METADATA', 'db'), 'MTLN_CDC_SRC_DATABASE'],\n [qualified_name('MTLN_CDC_EXT_METADATA', 'schema'), 'MTLN_CDC_SRC_SCHEMA'],\n [qualified_name('MTLN_CDC_EXT_METADATA', 'table'), 'MTLN_CDC_SRC_TABLE'],\n [qualified_name('MTLN_CDC_EXT_VERSION'), 'MTLN_CDC_SRC_VERSION'],\n [qualified_name('MTLN_CDC_EXT_METADATA', 'op'), 'MTLN_CDC_LAST_CHANGE_TYPE'],\n [qualified_name('MTLN_CDC_EXT_METADATA', 'ts_ms'), 'MTLN_CDC_LAST_COMMIT_TIMESTAMP'],\n [qualified_name('MTLN_CDC_EXT_DATE_HOUR'), 'MTLN_CDC_PROCESSED_DATE_HOUR'],\n [qualified_name('MTLN_CDC_EXT_FILENAME'), 'MTLN_CDC_FILENAME'],\n [qualified_name('MTLN_CDC_EXT_FILEPATH'), 'MTLN_CDC_FILEPATH'],\n [sequence_number_calc, 'MTLN_CDC_SEQUENCE_NUMBER'],\n [run_history_id, 'MTLN_CDC_LOAD_BATCH_ID'],\n ['CURRENT_TIMESTAMP', 'MTLN_CDC_LOAD_TIMESTAMP'],\n])\n\n# Add mtln_cdc_deleted flag for Copy Table With Soft Deletes transformations\nif transformation_type == 'copy table with soft deletes':\n calculated_columns.append([qualified_name('MTLN_CDC_DELETED'), 'MTLN_CDC_DELETED'])\n\nprint('calculated_columns:')\nfor row in calculated_columns:\n print(\"\\t\".join(str(col) for col in row))\ncontext.updateGridVariable('calculated_columns', calculated_columns)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44409":{"id":44409,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":1604,"y":1445,"width":32,"height":32,"inputConnectorIDs":[44412],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Failure - Transform and load soft deletes"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44417":{"id":44417,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":560,"y":720,"width":32,"height":32,"inputConnectorIDs":[44428],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Failure - Transform and append"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44444":{"id":44444,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":964,"y":1043,"width":32,"height":32,"inputConnectorIDs":[44447],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Failure - Transform and load"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44457":{"id":44457,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":1280,"y":1520,"width":32,"height":32,"inputConnectorIDs":[44462],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Failure - Transform and load without deletes"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"38666":{"id":38666,"sourceID":41529,"targetID":41538},"38688":{"id":38688,"sourceID":41538,"targetID":41539},"38705":{"id":38705,"sourceID":41541,"targetID":41542},"38721":{"id":38721,"sourceID":41534,"targetID":41535},"41548":{"id":41548,"sourceID":41532,"targetID":41533},"41552":{"id":41552,"sourceID":41531,"targetID":41533},"41555":{"id":41555,"sourceID":41527,"targetID":41537},"44393":{"id":44393,"sourceID":44233,"targetID":41540}},"failureConnectors":{"44412":{"id":44412,"sourceID":41538,"targetID":44409},"44428":{"id":44428,"sourceID":41541,"targetID":44417},"44447":{"id":44447,"sourceID":41534,"targetID":44444},"44462":{"id":44462,"sourceID":41529,"targetID":44457}},"unconditionalConnectors":{"38669":{"id":38669,"sourceID":41529,"targetID":38609},"38679":{"id":38679,"sourceID":41538,"targetID":33886},"38712":{"id":38712,"sourceID":41541,"targetID":33856},"38726":{"id":38726,"sourceID":41534,"targetID":33617},"41563":{"id":41563,"sourceID":41533,"targetID":41529},"44234":{"id":44234,"sourceID":41543,"targetID":44233}},"trueConnectors":{"41556":{"id":41556,"sourceID":41537,"targetID":41534},"41558":{"id":41558,"sourceID":41530,"targetID":41531},"49021":{"id":49021,"sourceID":41540,"targetID":41541}},"falseConnectors":{"41559":{"id":41559,"sourceID":41537,"targetID":41530},"41560":{"id":41560,"sourceID":41530,"targetID":41532},"41561":{"id":41561,"sourceID":41540,"targetID":41527}},"iterationConnectors":{},"noteConnectors":{},"canUndo":false,"undoCommand":"","undoCreated":0,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"41520":{"id":41520,"x":326,"y":806,"width":151,"height":229,"text":"**Build join_output grid var for Copy Table**\n\nIn the transformation jobs, we perform a left join onto the target table. Specify the output columns with the __join_output__grid variable.","colour":"e6e63c"},"41521":{"id":41521,"x":646,"y":1127,"width":468,"height":388,"text":"**Build delete_mapping**\n\nFor soft deletes, a __delete_mapping__grid variable is passed into the transformation job. This contains the columns which are to be updated when marking a record as deleted.","colour":"e6e63c"},"41522":{"id":41522,"x":-235,"y":485,"width":308,"height":229,"text":"**Build calculated_columns**\n\nIn the transformation job, we use a grid variable to configure a calculator component to\n- Extract fields from `before` or `after` objects\n- Add metadata columns into the transformation","colour":"e6e63c"},"41523":{"id":41523,"x":166,"y":485,"width":311,"height":230,"text":"**Transform and append change events**\n\nAppend latest change events to target table as change log\n","colour":"e6e63c"},"41524":{"id":41524,"x":1206,"y":1126,"width":309,"height":308,"text":"**Transform and load with soft deletes**\n\nApply latest CDC data to target table, updating MTLN_CDC_DELETED where records have been deleted from source.\n\nNote: The payload for a delete event does not contain an __after__object, and the __before__object only contains the primary key. If the current batch of change events contains an insert or update for a source record, as well as a delete, we must process that insert or update first. After processing the delete event, and marking the target record as deleted, the other columns in the target record should reflect the final value of the source record.","colour":"e6e63c"},"41525":{"id":41525,"x":564,"y":806,"width":310,"height":229,"text":"**Transform and load without soft deletes**\n\nApply latest CDC data to target table, deleting records which have been deleted from source","colour":"e6e63c"}},"variables":{"connector":{"definition":{"name":"connector","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"log_table":{"definition":{"name":"log_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"warehouse":{"definition":{"name":"warehouse","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"stage_table":{"definition":{"name":"stage_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"min_filename":{"definition":{"name":"min_filename","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"source_table":{"definition":{"name":"source_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"stage_schema":{"definition":{"name":"stage_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_table":{"definition":{"name":"target_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"min_date_hour":{"definition":{"name":"min_date_hour","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"source_schema":{"definition":{"name":"source_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_schema":{"definition":{"name":"target_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"commit_id_name":{"definition":{"name":"commit_id_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"source_version":{"definition":{"name":"source_version","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"append_metadata":{"definition":{"name":"append_metadata","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"commit_id_scale":{"definition":{"name":"commit_id_scale","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"join_expression":{"definition":{"name":"join_expression","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"logging_enabled":{"definition":{"name":"logging_enabled","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":null},"source_database":{"definition":{"name":"source_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_database":{"definition":{"name":"target_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"commit_id_datatype":{"definition":{"name":"commit_id_datatype","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"exported_from_name":{"definition":{"name":"exported_from_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"nested_select_list":{"definition":{"name":"nested_select_list","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"commit_id_precision":{"definition":{"name":"commit_id_precision","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"logged_rows_deleted":{"definition":{"name":"logged_rows_deleted","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"},"logged_rows_updated":{"definition":{"name":"logged_rows_updated","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"},"transformation_type":{"definition":{"name":"transformation_type","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"logged_rows_inserted":{"definition":{"name":"logged_rows_inserted","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"},"commit_id_database_type":{"definition":{"name":"commit_id_database_type","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""}},"grids":{"join_output":{"definition":{"name":"join_output","scope":"TASKBATCH","definitions":[{"name":"input_column","type":"TEXT"},{"name":"output_column","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[]},"primary_key":{"definition":{"name":"primary_key","scope":"TASKBATCH","definitions":[{"name":"key_column","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]},"delete_mapping":{"definition":{"name":"delete_mapping","scope":"TASKBATCH","definitions":[{"name":"column_name","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[]},"nested_metadata":{"definition":{"name":"nested_metadata","scope":"TASKBATCH","definitions":[{"name":"property","type":"TEXT"},{"name":"alias","type":"TEXT"},{"name":"format_string","type":"TEXT"},{"name":"type","type":"TEXT"},{"name":"size","type":"DECIMAL"},{"name":"precision","type":"DECIMAL"},{"name":"database_type","type":"TEXT"},{"name":"default_value","type":"TEXT"},{"name":"not_null","type":"TEXT"},{"name":"unique","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]},"target_metadata":{"definition":{"name":"target_metadata","scope":"TASKBATCH","definitions":[{"name":"column_name","type":"TEXT"},{"name":"type","type":"TEXT"},{"name":"size","type":"DECIMAL"},{"name":"precision","type":"DECIMAL"},{"name":"database_type","type":"TEXT"},{"name":"default_value","type":"TEXT"},{"name":"not_null","type":"TEXT"},{"name":"unique","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]},"calculated_columns":{"definition":{"name":"calculated_columns","scope":"TASKBATCH","definitions":[{"name":"calculation","type":"TEXT"},{"name":"alias","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[]}}},"Sync Single Table - 1-2-6 - Apply tags":{"id":-2,"revision":1,"created":1728917529394,"timestamp":1728917529394,"components":{"45920":{"id":45920,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":0,"y":0,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[46749],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Start","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"46721":{"id":46721,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":480,"y":80,"width":32,"height":32,"inputConnectorIDs":[46739],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[46738],"outputFalseConnectorIDs":[46740],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Protocol is gs?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"protocol"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"gs"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"protocol"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"gcs"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Or"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"46722":{"id":46722,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":800,"y":80,"width":32,"height":32,"inputConnectorIDs":[46738],"outputSuccessConnectorIDs":[46733],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Tag files - GCS"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"\"\"\"Set Custom-Time metadata on list of files in GCS.\"\"\"\nfrom datetime import datetime, timezone\nfrom google.cloud import storage\n\n\nbucket, _ = host_path.split('/', 1)\nprint(f'bucket: {bucket!r}')\n\ncustom_time = datetime.now(timezone.utc)\n\nprocessed_files = context.getGridVariable('processed_files') or []\n\nstorage_client = storage.Client()\nbucket_object = storage_client.bucket(bucket)\n\nfor [blob_name] in processed_files:\n\n print()\n print(f\"Reading 'Custom-Time' metadata for {blob_name!r}\")\n blob = bucket_object.get_blob(blob_name)\n if not blob:\n print('Blob does not exist')\n continue\n\n current_custom_time = blob.custom_time\n\n if current_custom_time and current_custom_time > custom_time:\n print(f\"Skipping, existing 'Custom-Time' value ({current_custom_time}) > new value ({custom_time})\")\n continue\n\n if not current_custom_time:\n print(f\"Setting 'Custom-Time' to {custom_time}\")\n elif blob_name[-11:] == 'schema.avsc':\n print(\"Skipping, 'schema.avsc' already has 'Custom-Time' set\")\n else:\n print(f\"Updating 'Custom-Time' from {current_custom_time} to {custom_time}\")\n\n blob.custom_time = custom_time\n blob.patch()\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"46723":{"id":46723,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":640,"y":160,"width":32,"height":32,"inputConnectorIDs":[46740],"outputSuccessConnectorIDs":[46736],"outputFailureConnectorIDs":[46742],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Validate Azure parameters"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"print(f'azure_key_vault_url: {azure_key_vault_url!r}')\nprint(f'storage_account_key_secret: {storage_account_key_secret!r}')\n\nmissing_parameters = []\n\nif not azure_key_vault_url:\n missing_parameters.append('Azure Key Vault URL')\n \nif not storage_account_key_secret:\n missing_parameters.append('Azure Blob Storage Account Key Secret')\n \nif missing_parameters:\n err_msg = ('The following shared job parameters were not set, but are '\n f'required for Azure Blob Storage: {missing_parameters}')\n raise Exception(err_msg)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"46724":{"id":46724,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":640,"y":320,"width":32,"height":32,"inputConnectorIDs":[46742],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Failure - Azure Blob Storage parameters not set"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"46725":{"id":46725,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":160,"y":0,"width":32,"height":32,"inputConnectorIDs":[46749],"outputSuccessConnectorIDs":[46734],"outputFailureConnectorIDs":[46743],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Identify storage protocol"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"# split the cloud_storage_url into protocol and host_path\nurl_list = cloud_storage_url.split('://', 1)\n\n# check we could split the string and extract a protocol\nif len(url_list) == 1:\n err_msg = f'Unable to determine the protocol from the cloud_storage_url {cloud_storage_url!r}'\n raise Exception(err_msg)\n\n# unpack the list\nprotocol, host_path = url_list\nprotocol = protocol.lower()\n\n# check protocol is supported\nsupported_protocols = ('azure', 's3', 'gs', 'gcs')\nif protocol not in supported_protocols:\n err_msg = f'The protocol {protocol!r} was identifed from the cloud_storage_url {cloud_storage_url!r}, but the only supported protocols are {\", \".join(supported_protocols)}'\n raise Exception(err_msg)\n\nprint(f'protocol: {protocol!r}')\ncontext.updateVariable('protocol', protocol)\n\nprint(f'host_path: {host_path!r}')\ncontext.updateVariable('host_path', host_path)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"46726":{"id":46726,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":160,"y":320,"width":32,"height":32,"inputConnectorIDs":[46743],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Failure - Storage protocol not supported"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"46727":{"id":46727,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":320,"y":0,"width":32,"height":32,"inputConnectorIDs":[46734],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[46737],"outputFalseConnectorIDs":[46739],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Protocol is s3?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"protocol"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"s3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"46728":{"id":46728,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":1120,"y":0,"width":32,"height":32,"inputConnectorIDs":[46741],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Success"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"46729":{"id":46729,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":800,"y":0,"width":32,"height":32,"inputConnectorIDs":[46737],"outputSuccessConnectorIDs":[46732],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Tag files - S3"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"\"\"\"Apply matillion-cdc-processed tag to list of files in S3.\"\"\"\nfrom boto3 import client as boto3_client\n\n\ndef get_tags(client, bucket, object_key):\n \"\"\"Call S3 API to retrieve current tags for object.\"\"\"\n try:\n response = client.get_object_tagging(\n Bucket=bucket,\n Key=object_key,\n )\n except client.exceptions.NoSuchKey:\n raise\n\n return response['TagSet']\n\n\ndef put_tags(client, bucket, object_key, tag_set):\n \"\"\"Call S3 API to set new tags against an object.\"\"\"\n client.put_object_tagging(\n Bucket=bucket,\n Key=object_key,\n Tagging={\n 'TagSet': tag_set\n },\n )\n\n\ndef available_tags(tag_set, tag_key, max_tags):\n \"\"\"Check whether the tag_set has enough room for an additional tag.\"\"\"\n if len(tag_set) < max_tags:\n return True\n\n # if we're at the max, check if our tag is already one of them\n for tag in tag_set:\n if tag['Key'] == tag_key:\n return True\n else:\n return False\n\n\ndef tag_already_exists(tag_set, tag_key, tag_value):\n \"\"\"Check if our tag key and value already exist in tag_set.\"\"\"\n for tag in tag_set:\n if tag['Key'] == tag_key and tag['Value'] == tag_value:\n return True\n return False\n\n\ndef update_tag_set(tag_set, tag_key, tag_value):\n \"\"\"Add or update our tag_key to tag_set.\"\"\"\n for tag in tag_set:\n if tag['Key'] == tag_key:\n tag['Value'] = tag_value\n break\n else:\n tag_set.append(\n {\n 'Key': tag_key,\n 'Value': tag_value\n }\n )\n\n\nclient = boto3_client('s3')\n\nbucket, _ = host_path.split('/', 1)\nprint(f'bucket: {bucket!r}')\n\ns3_max_tags = 10\ntag_key = 'matillion_cdc_processed'\ntag_value = 'true'\n\nprocessed_files = context.getGridVariable('processed_files') or []\n\nfor [object_key] in processed_files:\n\n print()\n print(f'Reading existing tags for {object_key!r}')\n try:\n tag_set = get_tags(client, bucket, object_key)\n except client.exceptions.NoSuchKey:\n print('Key does not exist')\n continue\n\n if not available_tags(tag_set, tag_key, s3_max_tags):\n print(f\"Skipping, the maximum number of tags ({s3_max_tags}) already exist for {object_key!r}\")\n continue\n\n if tag_already_exists(tag_set, tag_key, tag_value):\n print(f\"Skipping, {tag_key!r} tag already exists for {object_key!r}\")\n continue\n\n update_tag_set(tag_set, tag_key, tag_value)\n\n print(f\"Adding {tag_key!r} tag to {object_key!r}\")\n put_tags(client, bucket, object_key, tag_set)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"46730":{"id":46730,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":800,"y":160,"width":32,"height":32,"inputConnectorIDs":[46736],"outputSuccessConnectorIDs":[46735],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Tag files - Azure"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"\"\"\"Apply matillion-cdc-processed tag to list of files in Azure Blob Storage.\"\"\"\ntry:\n from azure.core.exceptions import ResourceNotFoundError\n from azure.identity import ManagedIdentityCredential\n from azure.keyvault.secrets import SecretClient\n from azure.storage.blob import BlobServiceClient\nexcept ModuleNotFoundError as e:\n err_msg = (\n 'Module not found. The following Python3 packages are '\n 'required: [azure-identity, azure-keyvault-secrets, azure-storage-blob]'\n )\n raise ModuleNotFoundError(err_msg) from e\n\n\ndef get_storage_account_key(\n azure_key_vault_url: str, storage_account_key_secret: str\n) -> str:\n \"\"\"Get Storage Account key from Key Vault.\n\n Args:\n azure_key_vault_url: URL for the Azure Key Vault\n storage_account_key_secret: Name of the secret storing the Storage Account Key\n\n Returns:\n str: Storage Account key\n \"\"\"\n credential = ManagedIdentityCredential()\n key_vault_client = SecretClient(\n vault_url=azure_key_vault_url, credential=credential\n )\n key_vault_secret = key_vault_client.get_secret(storage_account_key_secret)\n\n return str(key_vault_secret.value)\n\n\ndef available_tags(tags, tag_key, max_tags):\n \"\"\"Check whether the tags dict has enough room for an additional tag.\"\"\"\n if len(tags) < max_tags:\n return True\n\n # if we're at the max, check if our tag is already one of them\n if tag_key in tags.keys():\n return True\n else:\n return False\n\n\ndef tag_already_exists(tags, tag_key, tag_value):\n \"\"\"Check if our tag key and value already exist in tags.\"\"\"\n if tag_key in tags.keys() and tags[tag_key] == tag_value:\n return True\n else:\n return False\n\n\nazure_max_tags = 10\ntag_key = 'matillion_cdc_processed'\ntag_value = 'true'\n\naccount, container, _ = host_path.split('/', 2)\nprint(f'account: {account!r}')\nprint(f'container: {container!r}')\nprint()\n\nprint(\n f'Attempting to read the storage account key from the secret {storage_account_key_secret!r} '\n f'in the Key Vault {azure_key_vault_url!r}'\n)\naccount_key = get_storage_account_key(\n azure_key_vault_url, storage_account_key_secret\n)\nprint('Successfully read storage account key')\n\nblob_service_client = BlobServiceClient(\n account_url=f'https://{account}', credential=account_key\n)\n\n# Blob tags not supported on storage accounts with a hierarchical namespace\naccount_info = blob_service_client.get_account_information()\nif account_info.get('is_hns_enabled', False):\n err_msg = (\n f'The storage account {account!r} has a hierarchical namespace enabled. \\n'\n 'Blob Storage does not support Blob Index Tags on accounts with a hierarchical namespace enabled. \\n'\n 'For more details, see https://learn.microsoft.com/en-us/azure/storage/blobs/storage-feature-support-in-storage-accounts#standard-general-purpose-v2-accounts \\n'\n 'Please set the shared job parameter \"Tag Processed Files\" to \"N\" and re-run the job.'\n )\n raise Exception(err_msg)\n\nprocessed_files = context.getGridVariable('processed_files') or []\n\nfor [blob_name] in processed_files:\n\n blob_client = blob_service_client.get_blob_client(container, blob_name)\n\n print()\n print(f'Reading existing tags for {blob_name!r}')\n try:\n tags = blob_client.get_blob_tags()\n except ResourceNotFoundError:\n print('Blob does not exist')\n continue\n\n if not available_tags(tags, tag_key, azure_max_tags):\n print(f\"Skipping, the maximum number of tags ({azure_max_tags}) already exist for {blob_name!r}\")\n continue\n\n if tag_already_exists(tags, tag_key, tag_value):\n print(f\"Skipping, {tag_key!r} tag already exists for {blob_name!r}\")\n continue\n\n tags[tag_key] = tag_value\n\n print(f\"Adding {tag_key!r} tag to {blob_name!r}\")\n blob_client.set_blob_tags(tags)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"600"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"46731":{"id":46731,"inputCardinality":"MANY","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1343684451,"x":960,"y":0,"width":32,"height":32,"inputConnectorIDs":[46732,46733,46735],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[46741],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Join paths"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"46732":{"id":46732,"sourceID":46729,"targetID":46731},"46733":{"id":46733,"sourceID":46722,"targetID":46731},"46734":{"id":46734,"sourceID":46725,"targetID":46727},"46735":{"id":46735,"sourceID":46730,"targetID":46731},"46736":{"id":46736,"sourceID":46723,"targetID":46730}},"failureConnectors":{"46742":{"id":46742,"sourceID":46723,"targetID":46724},"46743":{"id":46743,"sourceID":46725,"targetID":46726}},"unconditionalConnectors":{"46741":{"id":46741,"sourceID":46731,"targetID":46728},"46749":{"id":46749,"sourceID":45920,"targetID":46725}},"trueConnectors":{"46737":{"id":46737,"sourceID":46727,"targetID":46729},"46738":{"id":46738,"sourceID":46721,"targetID":46722}},"falseConnectors":{"46739":{"id":46739,"sourceID":46727,"targetID":46721},"46740":{"id":46740,"sourceID":46721,"targetID":46723}},"iterationConnectors":{},"noteConnectors":{},"canUndo":false,"undoCommand":"","undoCreated":0,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"46744":{"id":46744,"x":86,"y":-153,"width":947,"height":388,"text":"**Apply tags to processed files**\n","colour":"e6e63c"}},"variables":{"protocol":{"definition":{"name":"protocol","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"host_path":{"definition":{"name":"host_path","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"cloud_storage_url":{"definition":{"name":"cloud_storage_url","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"azure_key_vault_url":{"definition":{"name":"azure_key_vault_url","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"storage_account_key_secret":{"definition":{"name":"storage_account_key_secret","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""}},"grids":{"processed_files":{"definition":{"name":"processed_files","scope":"TASKBATCH","definitions":[{"name":"filepath","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[{"values":[""]}]}}},"Sync Single Table - 1-2-4-1 - Alter postgresql sequence column":{"id":-2,"revision":1,"created":1728917529394,"timestamp":1728917529394,"components":{"44216":{"id":44216,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":1440,"y":0,"width":32,"height":32,"inputConnectorIDs":[44241],"outputSuccessConnectorIDs":[44246],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Update existing_target_metadata grid var with new data type"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"from decimal import Decimal\n\n\nexisting_target_metadata = context.getGridVariable('existing_target_metadata') or []\nupdated_metadata = []\n\nfor row in existing_target_metadata:\n if row[0] == 'MTLN_CDC_SEQUENCE_NUMBER':\n updated_metadata.append([\n row[0],\n 'VARCHAR',\n Decimal(16777216),\n Decimal(0),\n 'VARCHAR',\n '',\n 'No',\n 'No',\n ])\n else:\n updated_metadata.append(row)\n\nprint('Updated existing_target_metadata:')\nfor x in updated_metadata:\n print(\"\\t\".join(map(str,x)))\ncontext.updateGridVariable('existing_target_metadata', updated_metadata)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"54522":{"id":54522,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":-80,"y":0,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[54709],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Start","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"54685":{"id":54685,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":1600,"y":0,"width":32,"height":32,"inputConnectorIDs":[44246],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Success"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"54708":{"id":54708,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":80,"y":0,"width":32,"height":32,"inputConnectorIDs":[54709],"outputSuccessConnectorIDs":[55162],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Build SQL statements"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"# define suffix for renamed columns\nfrom datetime import datetime as dt\n\n\nold_column_suffix = dt.now().strftime('_%Y%m%d%H%M%S')\n\ntarget_table_name_full = f'\"{target_database}\".\"{target_schema}\".\"{target_table}\"'\nongoing_column = '\"MTLN_CDC_SEQUENCE_NUMBER\"'\nold_column = f'\"MTLN_CDC_SEQUENCE_NUMBER{old_column_suffix}\"'\n\nalter_table_sql = \\\nf\"\"\"ALTER TABLE {target_table_name_full} \n RENAME COLUMN {ongoing_column} TO {old_column};\nALTER TABLE {target_table_name_full} \n ADD COLUMN {ongoing_column} VARCHAR;\"\"\"\n\nprint(f'alter_table_sql: \\n{alter_table_sql}')\ncontext.updateVariable('alter_table_sql', alter_table_sql)\nprint()\n\npopulate_column_sql = \\\nf\"\"\"UPDATE {target_table_name_full} SET\n {ongoing_column} = CONCAT_WS(\n ':',\n TO_CHAR(0, '000000000000000x'),\n TO_CHAR(COALESCE({old_column}, 0), '000000000000000x')\n )\n\"\"\"\n\nprint(f'populate_column_sql: \\n{populate_column_sql}')\ncontext.updateVariable('populate_column_sql', populate_column_sql)\nprint()\n\nwarning_list = [\n f'Renaming column {ongoing_column} with type NUMBER to {old_column} in {target_table}',\n f'Adding column {ongoing_column} with type VARCHAR to {target_table}',\n]\nwarning_messages = '\\n'.join(warning_list)\n\nprint(f\"Warning messages: \\n{warning_messages}\")\ncontext.updateVariable('warning_messages', warning_messages)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"54717":{"id":54717,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-798585337,"x":640,"y":0,"width":32,"height":32,"inputConnectorIDs":[55269],"outputSuccessConnectorIDs":[55370],"outputFailureConnectorIDs":[55291],"outputUnconditionalConnectorIDs":[55340],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"Message","mapTo":"comp_message","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Alter MTLN_CDC_SEQUENCE_NUMBER on target table"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"SQL Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${alter_table_sql}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"54729":{"id":54729,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-798585337,"x":1040,"y":0,"width":32,"height":32,"inputConnectorIDs":[55370],"outputSuccessConnectorIDs":[44241],"outputFailureConnectorIDs":[55431],"outputUnconditionalConnectorIDs":[55437],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"Message","mapTo":"comp_message","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Populate new MTLN_CDC_SEQUENCE_NUMBER column"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"SQL Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${populate_column_sql}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"55190":{"id":55190,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":320,"y":0,"width":32,"height":32,"inputConnectorIDs":[55162],"outputSuccessConnectorIDs":[55227],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log warning messages"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"warning"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Metadata Change Detected"},"2":{"slot":2,"type":"STRING","value":"${warning_messages}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"55218":{"id":55218,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":480,"y":0,"width":32,"height":32,"inputConnectorIDs":[55227],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[55269],"outputFalseConnectorIDs":[55234],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Fix target data types?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"fix_target_data_types"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"Y"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"fix_target_data_types"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"Yes"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Or"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"55219":{"id":55219,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":480,"y":160,"width":32,"height":32,"inputConnectorIDs":[55234],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[55250],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Schema Drift Not Enabled"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"error"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Schema Drift"},"2":{"slot":2,"type":"STRING","value":"Not Enabled"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"55220":{"id":55220,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":480,"y":320,"width":32,"height":32,"inputConnectorIDs":[55250],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Failure - Unable to update MTLN_CDC_SEQUENCE_NUMBER on target table"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"55260":{"id":55260,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":640,"y":160,"width":32,"height":32,"inputConnectorIDs":[55291],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[55263],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log ALTER TABLE Error"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"error"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Schema Drift"},"2":{"slot":2,"type":"STRING","value":"${comp_message}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"55261":{"id":55261,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":800,"y":160,"width":32,"height":32,"inputConnectorIDs":[55340],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log ALTER TABLE SQL"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"info"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Schema Drift SQL"},"2":{"slot":2,"type":"STRING","value":"${alter_table_sql}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"55262":{"id":55262,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":640,"y":320,"width":32,"height":32,"inputConnectorIDs":[55263],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Failure - Error altering MTLN_CDC_SEQUENCE_NUMBER in target table"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"55380":{"id":55380,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":1040,"y":160,"width":32,"height":32,"inputConnectorIDs":[55431],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[55434],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log UPDATE Error"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"error"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Schema Drift"},"2":{"slot":2,"type":"STRING","value":"${comp_message}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"55381":{"id":55381,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":1040,"y":320,"width":32,"height":32,"inputConnectorIDs":[55434],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Failure - Error populating MTLN_CDC_SEQUENCE_NUMBER in target table"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"55382":{"id":55382,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":1200,"y":160,"width":32,"height":32,"inputConnectorIDs":[55437],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log UPDATE TABLE SQL"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"info"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Schema Drift SQL"},"2":{"slot":2,"type":"STRING","value":"${populate_column_sql}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"44241":{"id":44241,"sourceID":54729,"targetID":44216},"44246":{"id":44246,"sourceID":44216,"targetID":54685},"55162":{"id":55162,"sourceID":54708,"targetID":55190},"55227":{"id":55227,"sourceID":55190,"targetID":55218},"55370":{"id":55370,"sourceID":54717,"targetID":54729}},"failureConnectors":{"55291":{"id":55291,"sourceID":54717,"targetID":55260},"55431":{"id":55431,"sourceID":54729,"targetID":55380}},"unconditionalConnectors":{"54709":{"id":54709,"sourceID":54522,"targetID":54708},"55250":{"id":55250,"sourceID":55219,"targetID":55220},"55263":{"id":55263,"sourceID":55260,"targetID":55262},"55340":{"id":55340,"sourceID":54717,"targetID":55261},"55434":{"id":55434,"sourceID":55380,"targetID":55381},"55437":{"id":55437,"sourceID":54729,"targetID":55382}},"trueConnectors":{"55269":{"id":55269,"sourceID":55218,"targetID":54717}},"falseConnectors":{"55234":{"id":55234,"sourceID":55218,"targetID":55219}},"iterationConnectors":{},"noteConnectors":{},"canUndo":false,"undoCommand":"","undoCreated":0,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"44291":{"id":44291,"x":1366,"y":-154,"width":147,"height":227,"text":"**Update grid var with new data type**\n\nUpdate __existing_target_metadata__grid var with new data type to ensure standard schema drift code is not triggered.","colour":"e6e63c"},"54744":{"id":54744,"x":246,"y":-155,"width":630,"height":388,"text":"**Update MTLN_CDC_SEQUENCE_NUMBER in target table**\n\nAlter the target table\n- Rename existing MTLN_CDC_SEQUENCE_NUMBER (NUMBER) with timestamp suffix\n- Add new MTLN_CDC_SEQUENCE_NUMBER (VARCHAR)\n\n","colour":"e6e63c"},"55426":{"id":55426,"x":7,"y":-154,"width":146,"height":231,"text":"**Build SQL statements**\n\nBuild SQL to \n- Rename and add MTLN_CDC_SEQUENCE_NUMBER column in target table\n- Populate new column from old column","colour":"e6e63c"},"55440":{"id":55440,"x":965,"y":-157,"width":312,"height":394,"text":"**Populate new column from old column**\n\nNew column will contain values such as '0000000000000000:00000012653548d8', where '0000012653548d8' is the hexadecimal representation of the __lsn__from the old column","colour":"e6e63c"}},"variables":{"comp_message":{"definition":{"name":"comp_message","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"target_table":{"definition":{"name":"target_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_schema":{"definition":{"name":"target_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"alter_table_sql":{"definition":{"name":"alter_table_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"target_database":{"definition":{"name":"target_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"warning_messages":{"definition":{"name":"warning_messages","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"populate_column_sql":{"definition":{"name":"populate_column_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"fix_target_data_types":{"definition":{"name":"fix_target_data_types","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""}},"grids":{"existing_target_metadata":{"definition":{"name":"existing_target_metadata","scope":"TASKBATCH","definitions":[{"name":"column_name","type":"TEXT"},{"name":"type","type":"TEXT"},{"name":"size","type":"DECIMAL"},{"name":"precision","type":"DECIMAL"},{"name":"database_type","type":"TEXT"},{"name":"default_value","type":"TEXT"},{"name":"not_null","type":"TEXT"},{"name":"unique","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]}}},"Sync Single Table - 1-2-4 - Resolve metadata drift":{"id":-2,"revision":1,"created":1728917529394,"timestamp":1728917529394,"components":{"41371":{"id":41371,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":-1200,"y":-80,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[41381],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41372":{"id":41372,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":356330947,"x":-800,"y":-80,"width":32,"height":32,"inputConnectorIDs":[41380],"outputSuccessConnectorIDs":[41377],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Read current metadata for target table"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Grid Variable","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"existing_target_metadata"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Grid Variable Mapping","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Name"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"Type"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"Size"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"Precision"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"Database Type"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"Default Value"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"Not Null"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"Unique"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41373":{"id":41373,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":-640,"y":-80,"width":32,"height":32,"inputConnectorIDs":[41377],"outputSuccessConnectorIDs":[54573],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Print target metadata"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"existing_target_metadata = context.getGridVariable('existing_target_metadata') or []\nprint('existing_target_metadata:')\nfor x in existing_target_metadata:\n print(\"\\t\".join(map(str,x)))\nprint()\n\nlatest_target_metadata = context.getGridVariable('latest_target_metadata') or []\nprint('latest_target_metadata:')\nfor x in latest_target_metadata:\n print(\"\\t\".join(map(str,x)))\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41374":{"id":41374,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1877341091,"x":320,"y":-80,"width":32,"height":32,"inputConnectorIDs":[54471],"outputSuccessConnectorIDs":[41379],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Check metadata match"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"stage_table_metadata","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"latest_target_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"column_name"},"2":{"slot":2,"type":"GRID","value":"type"},"3":{"slot":3,"type":"GRID","value":"size"},"4":{"slot":4,"type":"GRID","value":"precision"},"5":{"slot":5,"type":"GRID","value":"database_type"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"target_table_metadata","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"existing_target_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"column_name"},"2":{"slot":2,"type":"GRID","value":"type"},"3":{"slot":3,"type":"GRID","value":"size"},"4":{"slot":4,"type":"GRID","value":"precision"},"5":{"slot":5,"type":"GRID","value":"database_type"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"target_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"target_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"target_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"target_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"target_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table}"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"target_platform","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Snowflake"}}}},"visible":true,"status":{},"columnNames":[]},"10":{"slot":10,"name":"fix_target_data_types","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${fix_target_data_types}"}}}},"visible":true,"status":{},"columnNames":[]},"11":{"slot":11,"name":"old_column_suffix","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"12":{"slot":12,"name":"info_messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"13":{"slot":13,"name":"warning_messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"14":{"slot":14,"name":"sql_to_execute","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"15":{"slot":15,"name":"comp_message","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"16":{"slot":16,"name":"comp_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41375":{"id":41375,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":480,"y":-80,"width":32,"height":32,"inputConnectorIDs":[41379],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Success"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41376":{"id":41376,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":-1040,"y":-80,"width":32,"height":32,"inputConnectorIDs":[41381],"outputSuccessConnectorIDs":[41380],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Set fix_target_data_types"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"if schema_drift_action == 'update target':\n fix_target_data_types = 'Y'\nelse:\n fix_target_data_types = 'N'\n \nprint(f'fix_target_data_types: {fix_target_data_types!r}')\ncontext.updateVariable('fix_target_data_types', fix_target_data_types)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"43211":{"id":43211,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":-405,"y":165,"width":32,"height":32,"inputConnectorIDs":[43214],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Failure - Incorrect data type for MTLN_CDC_SEQUENCE_NUMBER"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"54437":{"id":54437,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":-240,"y":-80,"width":32,"height":32,"inputConnectorIDs":[54626],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[54527],"outputFalseConnectorIDs":[54464],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Need to alter MTLN_CDC_SEQUENCE_NUMBER?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"alter_sequence_column"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"54461":{"id":54461,"inputCardinality":"MANY","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1343684451,"x":80,"y":-80,"width":32,"height":32,"inputConnectorIDs":[54464,54553],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[54471],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Join paths"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"54524":{"id":54524,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":-80,"y":0,"width":32,"height":32,"inputConnectorIDs":[54527],"outputSuccessConnectorIDs":[54553],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"gridExportMappings":{"1":{"slot":1,"fromGrid":"existing_target_metadata","toGrid":"existing_target_metadata","columnMapToFrom":{"database_type":"database_type","size":"size","not_null":"not_null","unique":"unique","precision":"precision","column_name":"column_name","default_value":"default_value","type":"type"},"validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Alter postgresql sequence column"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync Single Table - 1-2-4-1 - Alter postgresql sequence column"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"fix_target_data_types"},"2":{"slot":2,"type":"STRING","value":"${fix_target_data_types}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"target_database"},"2":{"slot":2,"type":"STRING","value":"${target_database}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"target_schema"},"2":{"slot":2,"type":"STRING","value":"${target_schema}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"target_table"},"2":{"slot":2,"type":"STRING","value":"${target_table}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"existing_target_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"column_name"},"2":{"slot":2,"type":"GRID","value":"type"},"3":{"slot":3,"type":"GRID","value":"size"},"4":{"slot":4,"type":"GRID","value":"precision"},"5":{"slot":5,"type":"GRID","value":"database_type"},"6":{"slot":6,"type":"GRID","value":"default_value"},"7":{"slot":7,"type":"GRID","value":"not_null"},"8":{"slot":8,"type":"GRID","value":"unique"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"existing_target_metadata"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sync Single Table (Exchange)"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Data Loader.CDC"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"3"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"54572":{"id":54572,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":-400,"y":-80,"width":32,"height":32,"inputConnectorIDs":[54573],"outputSuccessConnectorIDs":[54626],"outputFailureConnectorIDs":[43214],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Check data type of MTLN_CDC_ SEQUENCE_NUMBER"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"from decimal import Decimal\n\n\nif connector not in ('oracle', 'postgresql'):\n print(f\"Source connector is {connector!r}, so no need to check data type for 'mtln_cdc_sequence_number'\")\n exit()\n\nexisting_target_metadata = context.getGridVariable('existing_target_metadata') or []\nalter_sequence_column = Decimal(0)\n\nfor column, data_type, *_ in existing_target_metadata:\n if column == 'MTLN_CDC_SEQUENCE_NUMBER':\n if connector == 'postgresql':\n if data_type == 'NUMBER' and commit_id_name == 'sequence':\n alter_sequence_column = Decimal(1)\n print(\n \"The column 'MTLN_CDC_SEQUENCE_NUMBER' in the target table currently has type NUMBER(38, 0). \\n\"\n \"However, the new Avro files now contain 'metadata.sequence', so we need to alter this column \\n\"\n \"to VARCHAR. The existing data will be converted to the new format.\"\n )\n else:\n print(\"The data type for 'MTLN_CDC_SEQUENCE_NUMBER' is already correct in the target table\")\n else:\n if data_type == 'NUMBER':\n err_msg = (\n 'The target table was created with an earlier revision of this shared job.\\n'\n 'For Oracle pipelines, you must drop the stage and target tables and re-run this job'\n 'to re-create the tables.'\n )\n raise Exception(err_msg)\n else:\n print(\"The data type for 'MTLN_CDC_SEQUENCE_NUMBER' is correct in the target table\")\n break\n\nprint(f'alter_sequence_column: {alter_sequence_column!r}')\ncontext.updateVariable('alter_sequence_column', alter_sequence_column)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"41377":{"id":41377,"sourceID":41372,"targetID":41373},"41379":{"id":41379,"sourceID":41374,"targetID":41375},"41380":{"id":41380,"sourceID":41376,"targetID":41372},"54553":{"id":54553,"sourceID":54524,"targetID":54461},"54573":{"id":54573,"sourceID":41373,"targetID":54572},"54626":{"id":54626,"sourceID":54572,"targetID":54437}},"failureConnectors":{"43214":{"id":43214,"sourceID":54572,"targetID":43211}},"unconditionalConnectors":{"41381":{"id":41381,"sourceID":41371,"targetID":41376},"54471":{"id":54471,"sourceID":54461,"targetID":41374}},"trueConnectors":{"54527":{"id":54527,"sourceID":54437,"targetID":54524}},"falseConnectors":{"54464":{"id":54464,"sourceID":54437,"targetID":54461}},"iterationConnectors":{},"noteConnectors":{},"canUndo":false,"undoCommand":"","undoCreated":0,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"41368":{"id":41368,"x":-874,"y":-234,"width":309,"height":229,"text":"**Read current metadata for target table**","colour":"e6e63c"},"41369":{"id":41369,"x":244,"y":-234,"width":149,"height":228,"text":"**Check metadata match**\n\nUse shared library job to compare __latest_target_metadata__with __existing_target_metadata__\nOptionally update the target table so that it is compatible with __latest_target_metadata__","colour":"e6e63c"},"41370":{"id":41370,"x":-1113,"y":-235,"width":146,"height":229,"text":"**Set fix_target_data_types**\n\nUse shared job parameter __schema_drift_action__to set __fix_target_data_types__","colour":"e6e63c"},"54564":{"id":54564,"x":-474,"y":-236,"width":627,"height":309,"text":"**Check sequence number metadata**\n\nWith rev 3, we are using __metadata.sequence__to sort the change events for Postgres sources.\n- If the Avro files contain __metadata.sequence__, but the target table MTLN_CDC_SEQUENCE_NUMBER is numeric, alter the target table.\n\nFor Oracle, we are now using rs_id and ssn to generate a text MTLN_CDC_SEQUENCE_NUMBER column.\n- If a numeric column is detected in the target table, error the job execution","colour":"e6e63c"}},"variables":{"comp_name":{"definition":{"name":"comp_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"connector":{"definition":{"name":"connector","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"rows_added":{"definition":{"name":"rows_added","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"comp_message":{"definition":{"name":"comp_message","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"rows_removed":{"definition":{"name":"rows_removed","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"target_table":{"definition":{"name":"target_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":null},"info_messages":{"definition":{"name":"info_messages","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_schema":{"definition":{"name":"target_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":null},"commit_id_name":{"definition":{"name":"commit_id_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"sql_to_execute":{"definition":{"name":"sql_to_execute","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_database":{"definition":{"name":"target_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":null},"warning_messages":{"definition":{"name":"warning_messages","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"old_column_suffix":{"definition":{"name":"old_column_suffix","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"schema_drift_action":{"definition":{"name":"schema_drift_action","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"alter_sequence_column":{"definition":{"name":"alter_sequence_column","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"fix_target_data_types":{"definition":{"name":"fix_target_data_types","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"Y"},"target_table_name_full":{"definition":{"name":"target_table_name_full","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""}},"grids":{"latest_target_metadata":{"definition":{"name":"latest_target_metadata","scope":"TASKBATCH","definitions":[{"name":"column_name","type":"TEXT"},{"name":"type","type":"TEXT"},{"name":"size","type":"DECIMAL"},{"name":"precision","type":"DECIMAL"},{"name":"database_type","type":"TEXT"},{"name":"default_value","type":"TEXT"},{"name":"not_null","type":"TEXT"},{"name":"unique","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[{"values":["ITEM_ID","NUMBER","38","0","NUMBER(38, 0)","","No","No"]},{"values":["NAME","VARCHAR","16777216","0","VARCHAR(16777216)","","No","No"]},{"values":["COLOUR","VARCHAR","16777216","0","VARCHAR(16777216)","","No","No"]},{"values":["QUANTITY","NUMBER","38","0","NUMBER(38, 0)","","No","No"]},{"values":["LAST_ORDERED","NUMBER","38","0","NUMBER(38, 0)","","No","No"]},{"values":["MTLN_CDC_SRC_DATABASE","VARCHAR","16777216","0","VARCHAR(16777216)","","No","No"]},{"values":["MTLN_CDC_SRC_SCHEMA","VARCHAR","16777216","0","VARCHAR(16777216)","","No","No"]},{"values":["MTLN_CDC_SRC_TABLE","VARCHAR","16777216","0","VARCHAR(16777216)","","No","No"]},{"values":["MTLN_CDC_SRC_VERSION","NUMBER","38","0","NUMBER(38, 0)","","No","No"]},{"values":["MTLN_CDC_LAST_CHANGE_TYPE","VARCHAR","16777216","0","VARCHAR(16777216)","","No","No"]},{"values":["MTLN_CDC_LAST_COMMIT_TIMESTAMP","NUMBER","38","0","NUMBER(38, 0)","","No","No"]},{"values":["MTLN_CDC_PROCESSED_DATE_HOUR","TIMESTAMP","9","0","TIMESTAMP","","No","No"]},{"values":["MTLN_CDC_FILENAME","VARCHAR","16777216","0","VARCHAR(16777216)","","No","No"]},{"values":["MTLN_CDC_SEQUENCE_NUMBER","NUMBER","38","0","NUMBER(38, 0)","","No","No"]},{"values":["MTLN_CDC_LOAD_BATCH_ID","NUMBER","38","0","NUMBER(38, 0)","","No","No"]},{"values":["MTLN_CDC_LOAD_TIMESTAMP","TIMESTAMP","9","0","TIMESTAMP","","No","No"]}]},"existing_target_metadata":{"definition":{"name":"existing_target_metadata","scope":"TASKBATCH","definitions":[{"name":"column_name","type":"TEXT"},{"name":"type","type":"TEXT"},{"name":"size","type":"DECIMAL"},{"name":"precision","type":"DECIMAL"},{"name":"database_type","type":"TEXT"},{"name":"default_value","type":"TEXT"},{"name":"not_null","type":"TEXT"},{"name":"unique","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[]}}}},"transformationJobs":{"Sync Single Table - 1-2-5-3 - Transform and load without deletes":{"id":-2,"revision":1,"created":1728917529954,"timestamp":1728917529954,"components":{"41308":{"id":41308,"inputCardinality":"ZERO","outputCardinality":"MANY","implementationID":1354890871,"x":80,"y":160,"width":32,"height":32,"inputConnectorIDs":[],"outputConnectorIDs":[47247],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Stage table"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_table}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Column Names","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DATABASE"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"SCHEMA"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"TABLE"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"DATE_HOUR"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"FILENAME"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"VALUE"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"VERSION"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"FILEPATH"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Offset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"41310":{"id":41310,"inputCardinality":"ONE","outputCardinality":"MANY","implementationID":2139809017,"x":1040,"y":160,"width":32,"height":32,"inputConnectorIDs":[41324],"outputConnectorIDs":[41331],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Order changes for each key"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Include input columns.","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Yes"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Partition Data","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"primary_key"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"key_column"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Ordering within partitions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_SEQUENCE_NUMBER"},"2":{"slot":2,"type":"STRING","value":"Desc"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Functions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Row Number"},"2":{"slot":2,"type":"STRING","value":"MTLN_CDC_TEMP_KEY_ROW_NUM"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"41311":{"id":41311,"inputCardinality":"ONE","outputCardinality":"MANY","implementationID":-1760161015,"x":1200,"y":160,"width":32,"height":32,"inputConnectorIDs":[41331],"outputConnectorIDs":[41325],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Filter for last change only"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Filter Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_TEMP_KEY_ROW_NUM"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"AND"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"41312":{"id":41312,"inputCardinality":"ONE","outputCardinality":"ZERO","implementationID":-1848704949,"x":2016,"y":160,"width":32,"height":32,"inputConnectorIDs":[44765],"outputConnectorIDs":[],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Apply inserts and updates"}}}},"visible":true,"status":{},"columnNames":[]},"200":{"slot":200,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table}"}}}},"visible":true,"status":{},"columnNames":[]},"201":{"slot":201,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"202":{"slot":202,"name":"Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"203":{"slot":203,"name":"Warehouse","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${warehouse}"}}}},"visible":true,"status":{},"columnNames":[]},"204":{"slot":204,"name":"Target Alias","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"TARGET"}}}},"visible":true,"status":{},"columnNames":[]},"205":{"slot":205,"name":"Source Alias","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"SOURCE"}}}},"visible":true,"status":{},"columnNames":[]},"206":{"slot":206,"name":"Join Expression","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${join_expression}"},"2":{"slot":2,"type":"STRING","value":"Join"}}}},"visible":true,"status":{},"columnNames":[]},"207":{"slot":207,"name":"When Matched","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${join_expression}"},"2":{"slot":2,"type":"STRING","value":"Update"}}}},"visible":true,"status":{},"columnNames":[]},"208":{"slot":208,"name":"Update Mapping","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"target_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"column_name"},"2":{"slot":2,"type":"GRID","value":"column_name"}}}},"visible":true,"status":{},"columnNames":[]},"209":{"slot":209,"name":"Include Not Matched","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Yes"}}}},"visible":true,"status":{},"columnNames":[]},"210":{"slot":210,"name":"Insert Mapping","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"target_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"column_name"},"2":{"slot":2,"type":"GRID","value":"column_name"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"Rows Inserted","mapTo":"logged_rows_inserted","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"Rows Updated","mapTo":"logged_rows_updated","validationStatus":null,"validationMessage":null},"3":{"slot":3,"fromId":null,"fromName":"Rows Deleted","mapTo":"logged_rows_deleted","validationStatus":null,"validationMessage":null}},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"41313":{"id":41313,"inputCardinality":"ONE","outputCardinality":"MANY","implementationID":1716658327,"x":480,"y":160,"width":32,"height":32,"inputConnectorIDs":[47201],"outputConnectorIDs":[44707],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Extract payload and metadata objects"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Include Input Columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"No"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Calculations","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"\"VALUE\":\"after\""},"2":{"slot":2,"type":"STRING","value":"MTLN_CDC_EXT_PAYLOAD"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"\"VALUE\":\"metadata\""},"2":{"slot":2,"type":"STRING","value":"MTLN_CDC_EXT_METADATA"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"\"VERSION\""},"2":{"slot":2,"type":"STRING","value":"MTLN_CDC_EXT_VERSION"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"\"DATE_HOUR\""},"2":{"slot":2,"type":"STRING","value":"MTLN_CDC_EXT_DATE_HOUR"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"\"FILENAME\""},"2":{"slot":2,"type":"STRING","value":"MTLN_CDC_EXT_FILENAME"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"\"FILEPATH\""},"2":{"slot":2,"type":"STRING","value":"MTLN_CDC_EXT_FILEPATH"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"FALSE"},"2":{"slot":2,"type":"STRING","value":"MTLN_CDC_DELETED"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"41316":{"id":41316,"inputCardinality":"ONE","outputCardinality":"MANY","implementationID":-1760161015,"x":880,"y":160,"width":32,"height":32,"inputConnectorIDs":[44713],"outputConnectorIDs":[41324],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Filter out deletes"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Filter Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_LAST_CHANGE_TYPE"},"2":{"slot":2,"type":"STRING","value":"Not"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"d"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"OR"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"41317":{"id":41317,"inputCardinality":"ZERO","outputCardinality":"MANY","implementationID":1354890871,"x":1440,"y":240,"width":32,"height":32,"inputConnectorIDs":[],"outputConnectorIDs":[41328],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Target table"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Column Names","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"target_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"column_name"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Offset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"41318":{"id":41318,"inputCardinality":"MANY","outputCardinality":"MANY","implementationID":-629958239,"x":1600,"y":160,"width":32,"height":32,"inputConnectorIDs":[41328,41325],"outputConnectorIDs":[41327],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Join existing data"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Main Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Filter for last change only"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Main Table Alias","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"SOURCE"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Joins","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Target table"},"2":{"slot":2,"type":"STRING","value":"TARGET"},"3":{"slot":3,"type":"STRING","value":"Left"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Join Expressions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${join_expression}"},"2":{"slot":2,"type":"STRING","value":"SOURCE_Left_TARGET"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Output Columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"join_output"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"input_column"},"2":{"slot":2,"type":"GRID","value":"output_column"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"41319":{"id":41319,"inputCardinality":"ONE","outputCardinality":"MANY","implementationID":-1760161015,"x":1760,"y":160,"width":32,"height":32,"inputConnectorIDs":[41327],"outputConnectorIDs":[44765],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Filter for new data with newer MTLN_CDC_SEQUENCE_NUMBER"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Filter Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_TARGET_KEY"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Null"},"4":{"slot":4,"type":"STRING","value":""}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_TARGET_MTLN_CDC_SEQUENCE_NUMBER"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Less than"},"4":{"slot":4,"type":"STRING","value":"MTLN_CDC_SEQUENCE_NUMBER"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"OR"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"44671":{"id":44671,"inputCardinality":"ONE","outputCardinality":"MANY","implementationID":1716658327,"x":640,"y":160,"width":32,"height":32,"inputConnectorIDs":[44707],"outputConnectorIDs":[44713],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Extract nested columns"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Include Input Columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"No"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Calculations","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"calculated_columns"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"calculation"},"2":{"slot":2,"type":"GRID","value":"alias"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"47237":{"id":47237,"inputCardinality":"ZERO_OR_MANY","outputCardinality":"MANY","implementationID":-1266674941,"x":240,"y":160,"width":32,"height":32,"inputConnectorIDs":[47247],"outputConnectorIDs":[47201],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Filter on source table"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"SQL Query","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"SELECT \n * \nFROM $T{Stage table}\nWHERE \n \"DATABASE\" = '${source_database}'\n AND \"SCHEMA\" = '${source_schema}'\n AND \"TABLE\" = '${source_table}'\n AND \"VERSION\" = ${source_version}\n AND (\n (\n \"DATE_HOUR\" = '${min_date_hour}'\n AND \"FILENAME\" >= '${min_filename}'\n )\n OR\n (\n \"DATE_HOUR\" > '${min_date_hour}'\n )\n )"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"}},"connectors":{"41324":{"id":41324,"sourceID":41316,"targetID":41310},"41325":{"id":41325,"sourceID":41311,"targetID":41318},"41327":{"id":41327,"sourceID":41318,"targetID":41319},"41328":{"id":41328,"sourceID":41317,"targetID":41318},"41331":{"id":41331,"sourceID":41310,"targetID":41311},"44707":{"id":44707,"sourceID":41313,"targetID":44671},"44713":{"id":44713,"sourceID":44671,"targetID":41316},"44765":{"id":44765,"sourceID":41319,"targetID":41312},"47201":{"id":47201,"sourceID":47237,"targetID":41313},"47247":{"id":47247,"sourceID":41308,"targetID":47237}},"canUndo":false,"undoCommand":"","undoCreated":0,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"41301":{"id":41301,"x":406,"y":86,"width":308,"height":148,"text":"Extract columns","colour":"e6e63c"},"41302":{"id":41302,"x":807,"y":86,"width":470,"height":147,"text":"Find last change for each primary key which is not a delete","colour":"e6e63c"},"41305":{"id":41305,"x":1926,"y":87,"width":151,"height":149,"text":"Apply insert and updates","colour":"e6e63c"},"41306":{"id":41306,"x":5,"y":-153,"width":547,"height":149,"text":"**Transformation type = copy table with soft deletes**\n\nFirst transformation job for this transformation type\n\nExtract latest change events for source table, ignoring deletes, and apply the last change per primary key value to the target table\n\nThis ensures that the target table will contain the latest data prior to being marked as deleted","colour":"00ce4f"},"41307":{"id":41307,"x":1367,"y":86,"width":466,"height":226,"text":"Remove any change events where, for a given key, the MTLN_CDC_SEQUENCE_NUMBER is older than the current value","colour":"e6e63c"},"41350":{"id":41350,"x":5,"y":86,"width":308,"height":147,"text":"Read data from stage table, filter for source table and files newer than __max_filename__","colour":"e6e63c"},"44661":{"id":44661,"x":164,"y":247,"width":148,"height":145,"text":"Using a SQL component as the Filter component does not support the nested filtering we require on the __DATE_HOUR__ and __FILENAME__ columns.","colour":"f9c21b"}},"noteConnectors":{},"variables":{"warehouse":{"definition":{"name":"warehouse","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"stage_table":{"definition":{"name":"stage_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"min_filename":{"definition":{"name":"min_filename","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"source_table":{"definition":{"name":"source_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"stage_schema":{"definition":{"name":"stage_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_table":{"definition":{"name":"target_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"min_date_hour":{"definition":{"name":"min_date_hour","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"source_schema":{"definition":{"name":"source_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_schema":{"definition":{"name":"target_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"source_version":{"definition":{"name":"source_version","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"join_expression":{"definition":{"name":"join_expression","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"source_database":{"definition":{"name":"source_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_database":{"definition":{"name":"target_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"logged_rows_deleted":{"definition":{"name":"logged_rows_deleted","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"},"logged_rows_updated":{"definition":{"name":"logged_rows_updated","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"},"logged_rows_inserted":{"definition":{"name":"logged_rows_inserted","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"}},"grids":{"join_output":{"definition":{"name":"join_output","scope":"TASKBATCH","definitions":[{"name":"input_column","type":"TEXT"},{"name":"output_column","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]},"primary_key":{"definition":{"name":"primary_key","scope":"TASKBATCH","definitions":[{"name":"key_column","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]},"target_metadata":{"definition":{"name":"target_metadata","scope":"TASKBATCH","definitions":[{"name":"column_name","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]},"calculated_columns":{"definition":{"name":"calculated_columns","scope":"TASKBATCH","definitions":[{"name":"calculation","type":"TEXT"},{"name":"alias","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]}}},"Sync Single Table - 1-2-5-1 - Transform and append":{"id":-2,"revision":1,"created":1728917529954,"timestamp":1728917529954,"components":{"41231":{"id":41231,"inputCardinality":"ZERO","outputCardinality":"MANY","implementationID":1354890871,"x":240,"y":160,"width":32,"height":32,"inputConnectorIDs":[],"outputConnectorIDs":[47150],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Stage table"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_table}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Column Names","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DATABASE"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"SCHEMA"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"TABLE"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"DATE_HOUR"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"FILENAME"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"VALUE"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"VERSION"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"FILEPATH"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Offset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"41233":{"id":41233,"inputCardinality":"ONE","outputCardinality":"MANY","implementationID":1716658327,"x":640,"y":160,"width":32,"height":32,"inputConnectorIDs":[47182],"outputConnectorIDs":[44503],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Extract payload and metadata objects"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Include Input Columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"No"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Calculations","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"CASE \"VALUE\":\"metadata\".\"op\"\n WHEN 'd' THEN \"VALUE\":\"before\"\n ELSE \"VALUE\":\"after\"\nEND"},"2":{"slot":2,"type":"STRING","value":"MTLN_CDC_EXT_PAYLOAD"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"\"VALUE\":\"metadata\""},"2":{"slot":2,"type":"STRING","value":"MTLN_CDC_EXT_METADATA"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"\"VERSION\""},"2":{"slot":2,"type":"STRING","value":"MTLN_CDC_EXT_VERSION"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"\"DATE_HOUR\""},"2":{"slot":2,"type":"STRING","value":"MTLN_CDC_EXT_DATE_HOUR"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"\"FILENAME\""},"2":{"slot":2,"type":"STRING","value":"MTLN_CDC_EXT_FILENAME"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"\"FILEPATH\""},"2":{"slot":2,"type":"STRING","value":"MTLN_CDC_EXT_FILEPATH"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"41235":{"id":41235,"inputCardinality":"ONE","outputCardinality":"ZERO","implementationID":211954775,"x":1040,"y":160,"width":32,"height":32,"inputConnectorIDs":[44526],"outputConnectorIDs":[],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Append change events"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Column Mapping","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"target_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"column_name"},"2":{"slot":2,"type":"GRID","value":"column_name"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Truncate","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Append"}}}},"visible":true,"status":{},"columnNames":[]},"200":{"slot":200,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"201":{"slot":201,"name":"Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"202":{"slot":202,"name":"Warehouse","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${warehouse}"}}}},"visible":true,"status":{},"columnNames":[]},"203":{"slot":203,"name":"Fix Data Type Mismatches","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"No"}}}},"visible":true,"status":{},"columnNames":[]},"204":{"slot":204,"name":"Order By","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_SEQUENCE_NUMBER"},"2":{"slot":2,"type":"STRING","value":"Ascending"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"Row Count","mapTo":"logged_rows_inserted","validationStatus":null,"validationMessage":null}},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"44481":{"id":44481,"inputCardinality":"ONE","outputCardinality":"MANY","implementationID":1716658327,"x":800,"y":160,"width":32,"height":32,"inputConnectorIDs":[44503],"outputConnectorIDs":[44526],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Extract nested columns"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Include Input Columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"No"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Calculations","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"calculated_columns"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"calculation"},"2":{"slot":2,"type":"GRID","value":"alias"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"47149":{"id":47149,"inputCardinality":"ZERO_OR_MANY","outputCardinality":"MANY","implementationID":-1266674941,"x":400,"y":160,"width":32,"height":32,"inputConnectorIDs":[47150],"outputConnectorIDs":[47182],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Filter on source table"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"SQL Query","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"SELECT \n * \nFROM $T{Stage table}\nWHERE \n \"DATABASE\" = '${source_database}'\n AND \"SCHEMA\" = '${source_schema}'\n AND \"TABLE\" = '${source_table}'\n AND \"VERSION\" = ${source_version}\n AND (\n (\n \"DATE_HOUR\" = '${min_date_hour}'\n AND \"FILENAME\" >= '${min_filename}'\n )\n OR\n (\n \"DATE_HOUR\" > '${min_date_hour}'\n )\n )"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"}},"connectors":{"44503":{"id":44503,"sourceID":41233,"targetID":44481},"44526":{"id":44526,"sourceID":44481,"targetID":41235},"47150":{"id":47150,"sourceID":41231,"targetID":47149},"47182":{"id":47182,"sourceID":47149,"targetID":41233}},"canUndo":false,"undoCommand":"","undoCreated":0,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"41225":{"id":41225,"x":166,"y":87,"width":308,"height":147,"text":"Read data from stage table, filter for source table and files newer than __max_filename__","colour":"e6e63c"},"41226":{"id":41226,"x":565,"y":87,"width":308,"height":148,"text":"Extract columns","colour":"e6e63c"},"41227":{"id":41227,"x":967,"y":86,"width":150,"height":149,"text":"Append new change events to target table","colour":"e6e63c"},"41229":{"id":41229,"x":165,"y":-154,"width":547,"height":149,"text":"**Transformation type = change log**\n\nExtract latest change events for source table, and append them all to the target table as a change log","colour":"00ce4f"},"44566":{"id":44566,"x":327,"y":248,"width":148,"height":145,"text":"Using a SQL component as the Filter component does not support the nested filtering we require on the __DATE_HOUR__ and __FILENAME__ columns.","colour":"f9c21b"}},"noteConnectors":{},"variables":{"warehouse":{"definition":{"name":"warehouse","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"stage_table":{"definition":{"name":"stage_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"min_filename":{"definition":{"name":"min_filename","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"source_table":{"definition":{"name":"source_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"stage_schema":{"definition":{"name":"stage_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_table":{"definition":{"name":"target_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"min_date_hour":{"definition":{"name":"min_date_hour","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"source_schema":{"definition":{"name":"source_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_schema":{"definition":{"name":"target_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"source_version":{"definition":{"name":"source_version","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"source_database":{"definition":{"name":"source_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_database":{"definition":{"name":"target_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"logged_rows_deleted":{"definition":{"name":"logged_rows_deleted","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"},"logged_rows_updated":{"definition":{"name":"logged_rows_updated","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"},"logged_rows_inserted":{"definition":{"name":"logged_rows_inserted","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"}},"grids":{"target_metadata":{"definition":{"name":"target_metadata","scope":"TASKBATCH","definitions":[{"name":"column_name","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]},"calculated_columns":{"definition":{"name":"calculated_columns","scope":"TASKBATCH","definitions":[{"name":"calculation","type":"TEXT"},{"name":"alias","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]}}},"Logging - 1-1 - Append Metrics":{"id":-2,"revision":1,"created":1728917529954,"timestamp":1728917529954,"components":{"34888":{"id":34888,"inputCardinality":"ONE","outputCardinality":"ZERO","implementationID":211954775,"x":640,"y":320,"width":32,"height":32,"inputConnectorIDs":[34891],"outputConnectorIDs":[],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Append to Platform Metrics Store"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${log_table}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Column Mapping","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"table_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"column_name"},"2":{"slot":2,"type":"GRID","value":"column_name"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Truncate","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Append"}}}},"visible":true,"status":{},"columnNames":[]},"200":{"slot":200,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${schema}"}}}},"visible":true,"status":{},"columnNames":[]},"201":{"slot":201,"name":"Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${database}"}}}},"visible":true,"status":{},"columnNames":[]},"202":{"slot":202,"name":"Warehouse","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${warehouse}"}}}},"visible":true,"status":{},"columnNames":[]},"203":{"slot":203,"name":"Fix Data Type Mismatches","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"No"}}}},"visible":true,"status":{},"columnNames":[]},"204":{"slot":204,"name":"Order By","elements":{},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"34889":{"id":34889,"inputCardinality":"ZERO","outputCardinality":"MANY","implementationID":738944574,"x":320,"y":320,"width":32,"height":32,"inputConnectorIDs":[],"outputConnectorIDs":[34892],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Generate Record from Grid Variable"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"table_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"column_name"},"2":{"slot":2,"type":"GRID","value":"data_type"},"3":{"slot":3,"type":"GRID","value":"size"},"4":{"slot":4,"type":"GRID","value":"precision"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Values","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"fixed_flow_data"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"task_batch_id"},"2":{"slot":2,"type":"GRID","value":"timestamp"},"3":{"slot":3,"type":"GRID","value":"componet_name"},"4":{"slot":4,"type":"GRID","value":"namespace"},"5":{"slot":5,"type":"GRID","value":"key"},"6":{"slot":6,"type":"GRID","value":"value"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"34890":{"id":34890,"inputCardinality":"ONE","outputCardinality":"MANY","implementationID":1716658327,"x":480,"y":320,"width":32,"height":32,"inputConnectorIDs":[34892],"outputConnectorIDs":[34891],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Add Timestamp"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Include Input Columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Yes"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Calculations","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"CURRENT_TIMESTAMP(6)"},"2":{"slot":2,"type":"STRING","value":"TIMESTAMP"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"}},"connectors":{"34891":{"id":34891,"sourceID":34890,"targetID":34888},"34892":{"id":34892,"sourceID":34889,"targetID":34890}},"canUndo":false,"undoCommand":"","undoCreated":0,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"34887":{"id":34887,"x":248,"y":88,"width":466,"height":310,"text":"This transformation job will take the variables passed and update a fixed flow record to append to the target table as defined by ${metrics_table_name} in ${schema}.${database}. \n\nCURRENT_TIMESTAMP replaces the timestamp column with the current value.","colour":"e6e63c"}},"noteConnectors":{},"variables":{"schema":{"definition":{"name":"schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":null},"database":{"definition":{"name":"database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":null},"log_table":{"definition":{"name":"log_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":null},"warehouse":{"definition":{"name":"warehouse","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""}},"grids":{"table_metadata":{"definition":{"name":"table_metadata","scope":"BRANCH","definitions":[{"name":"column_name","type":"TEXT"},{"name":"data_type","type":"TEXT"},{"name":"size","type":"TEXT"},{"name":"precision","type":"TEXT"},{"name":"default_value","type":"TEXT"},{"name":"not_null","type":"TEXT"},{"name":"unique","type":"TEXT"},{"name":"comment","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]},"fixed_flow_data":{"definition":{"name":"fixed_flow_data","scope":"TASKBATCH","definitions":[{"name":"componet_name","type":"TEXT"},{"name":"task_batch_id","type":"TEXT"},{"name":"value","type":"TEXT"},{"name":"namespace","type":"TEXT"},{"name":"key","type":"TEXT"},{"name":"timestamp","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]}}},"Sync Single Table - 1-2-5-4 - Transform and load soft deletes":{"id":-2,"revision":1,"created":1728917529954,"timestamp":1728917529954,"components":{"41209":{"id":41209,"inputCardinality":"ZERO","outputCardinality":"MANY","implementationID":1354890871,"x":240,"y":160,"width":32,"height":32,"inputConnectorIDs":[],"outputConnectorIDs":[47216],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Stage table"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_table}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Column Names","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DATABASE"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"SCHEMA"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"TABLE"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"DATE_HOUR"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"FILENAME"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"VALUE"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"VERSION"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"FILEPATH"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Offset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"41211":{"id":41211,"inputCardinality":"ONE","outputCardinality":"MANY","implementationID":2139809017,"x":1040,"y":160,"width":32,"height":32,"inputConnectorIDs":[44726],"outputConnectorIDs":[41258],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Order changes for each key"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Include input columns.","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Yes"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Partition Data","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"primary_key"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"key_column"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Ordering within partitions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_SEQUENCE_NUMBER"},"2":{"slot":2,"type":"STRING","value":"Desc"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Functions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Row Number"},"2":{"slot":2,"type":"STRING","value":"MTLN_CDC_TEMP_KEY_ROW_NUM"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"41212":{"id":41212,"inputCardinality":"ONE","outputCardinality":"MANY","implementationID":-1760161015,"x":1200,"y":160,"width":32,"height":32,"inputConnectorIDs":[41258],"outputConnectorIDs":[41266],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Filter for last change if it's a delete"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Filter Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_TEMP_KEY_ROW_NUM"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"1"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_LAST_CHANGE_TYPE"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"d"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"AND"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"41213":{"id":41213,"inputCardinality":"ONE","outputCardinality":"MANY","implementationID":1716658327,"x":640,"y":160,"width":32,"height":32,"inputConnectorIDs":[47220],"outputConnectorIDs":[44723],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Extract payload and metadata objects"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Include Input Columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"No"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Calculations","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"CASE \"VALUE\":\"metadata\".\"op\"\n WHEN 'd' THEN \"VALUE\":\"before\"\n ELSE \"VALUE\":\"after\"\nEND"},"2":{"slot":2,"type":"STRING","value":"MTLN_CDC_EXT_PAYLOAD"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"\"VALUE\":\"metadata\""},"2":{"slot":2,"type":"STRING","value":"MTLN_CDC_EXT_METADATA"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"\"VERSION\""},"2":{"slot":2,"type":"STRING","value":"MTLN_CDC_EXT_VERSION"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"\"DATE_HOUR\""},"2":{"slot":2,"type":"STRING","value":"MTLN_CDC_EXT_DATE_HOUR"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"\"FILENAME\""},"2":{"slot":2,"type":"STRING","value":"MTLN_CDC_EXT_FILENAME"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"\"FILEPATH\""},"2":{"slot":2,"type":"STRING","value":"MTLN_CDC_EXT_FILEPATH"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"TRUE"},"2":{"slot":2,"type":"STRING","value":"MTLN_CDC_DELETED"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"41251":{"id":41251,"inputCardinality":"ONE","outputCardinality":"ZERO","implementationID":-1848704949,"x":2000,"y":160,"width":32,"height":32,"inputConnectorIDs":[44774],"outputConnectorIDs":[],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Apply soft deletes"}}}},"visible":true,"status":{},"columnNames":[]},"200":{"slot":200,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table}"}}}},"visible":true,"status":{},"columnNames":[]},"201":{"slot":201,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"202":{"slot":202,"name":"Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"203":{"slot":203,"name":"Warehouse","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${warehouse}"}}}},"visible":true,"status":{},"columnNames":[]},"204":{"slot":204,"name":"Target Alias","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"TARGET"}}}},"visible":true,"status":{},"columnNames":[]},"205":{"slot":205,"name":"Source Alias","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"SOURCE"}}}},"visible":true,"status":{},"columnNames":[]},"206":{"slot":206,"name":"Join Expression","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${join_expression}"},"2":{"slot":2,"type":"STRING","value":"Join"}}}},"visible":true,"status":{},"columnNames":[]},"207":{"slot":207,"name":"When Matched","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${join_expression}"},"2":{"slot":2,"type":"STRING","value":"Update"}}}},"visible":true,"status":{},"columnNames":[]},"208":{"slot":208,"name":"Update Mapping","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"delete_mapping"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"column_name"},"2":{"slot":2,"type":"GRID","value":"column_name"}}}},"visible":true,"status":{},"columnNames":[]},"209":{"slot":209,"name":"Include Not Matched","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"No"}}}},"visible":true,"status":{},"columnNames":[]},"210":{"slot":210,"name":"Insert Mapping","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"target_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"column_name"},"2":{"slot":2,"type":"GRID","value":"column_name"}}}},"visible":false,"status":{},"columnNames":[]}},"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"Rows Updated","mapTo":"logged_rows_updated","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"Rows Inserted","mapTo":"logged_rows_inserted","validationStatus":null,"validationMessage":null},"3":{"slot":3,"fromId":null,"fromName":"Rows Deleted","mapTo":"logged_rows_deleted","validationStatus":null,"validationMessage":null}},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"41252":{"id":41252,"inputCardinality":"ZERO","outputCardinality":"MANY","implementationID":1354890871,"x":1456,"y":240,"width":32,"height":32,"inputConnectorIDs":[],"outputConnectorIDs":[41264],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Target table"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Column Names","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"target_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"column_name"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Offset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"41253":{"id":41253,"inputCardinality":"MANY","outputCardinality":"MANY","implementationID":-629958239,"x":1600,"y":160,"width":32,"height":32,"inputConnectorIDs":[41264,41266],"outputConnectorIDs":[41265],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Join existing data"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Main Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Filter for last change if it's a delete"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Main Table Alias","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"SOURCE"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Joins","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Target table"},"2":{"slot":2,"type":"STRING","value":"TARGET"},"3":{"slot":3,"type":"STRING","value":"Left"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Join Expressions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${join_expression}"},"2":{"slot":2,"type":"STRING","value":"SOURCE_Left_TARGET"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Output Columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"join_output"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"input_column"},"2":{"slot":2,"type":"GRID","value":"output_column"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"41254":{"id":41254,"inputCardinality":"ONE","outputCardinality":"MANY","implementationID":-1760161015,"x":1760,"y":160,"width":32,"height":32,"inputConnectorIDs":[41265],"outputConnectorIDs":[44774],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Filter for new data with newer MTLN_CDC_SEQUENCE_NUMBER"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Filter Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_TARGET_KEY"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Null"},"4":{"slot":4,"type":"STRING","value":""}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_TARGET_MTLN_CDC_SEQUENCE_NUMBER"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Less than"},"4":{"slot":4,"type":"STRING","value":"MTLN_CDC_SEQUENCE_NUMBER"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"OR"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"44720":{"id":44720,"inputCardinality":"ONE","outputCardinality":"MANY","implementationID":1716658327,"x":800,"y":160,"width":32,"height":32,"inputConnectorIDs":[44723],"outputConnectorIDs":[44726],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Extract nested columns"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Include Input Columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"No"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Calculations","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"calculated_columns"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"calculation"},"2":{"slot":2,"type":"GRID","value":"alias"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"47206":{"id":47206,"inputCardinality":"ZERO_OR_MANY","outputCardinality":"MANY","implementationID":-1266674941,"x":400,"y":160,"width":32,"height":32,"inputConnectorIDs":[47216],"outputConnectorIDs":[47220],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Filter on source table"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"SQL Query","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"SELECT \n * \nFROM $T{Stage table}\nWHERE \n \"DATABASE\" = '${source_database}'\n AND \"SCHEMA\" = '${source_schema}'\n AND \"TABLE\" = '${source_table}'\n AND \"VERSION\" = ${source_version}\n AND (\n (\n \"DATE_HOUR\" = '${min_date_hour}'\n AND \"FILENAME\" >= '${min_filename}'\n )\n OR\n (\n \"DATE_HOUR\" > '${min_date_hour}'\n )\n )"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"}},"connectors":{"41258":{"id":41258,"sourceID":41211,"targetID":41212},"41264":{"id":41264,"sourceID":41252,"targetID":41253},"41265":{"id":41265,"sourceID":41253,"targetID":41254},"41266":{"id":41266,"sourceID":41212,"targetID":41253},"44723":{"id":44723,"sourceID":41213,"targetID":44720},"44726":{"id":44726,"sourceID":44720,"targetID":41211},"44774":{"id":44774,"sourceID":41254,"targetID":41251},"47216":{"id":47216,"sourceID":41209,"targetID":47206},"47220":{"id":47220,"sourceID":47206,"targetID":41213}},"canUndo":false,"undoCommand":"","undoCreated":0,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"41201":{"id":41201,"x":166,"y":86,"width":308,"height":147,"text":"Read data from stage table, filter for source table and files newer than __max_filename__","colour":"e6e63c"},"41202":{"id":41202,"x":565,"y":86,"width":308,"height":148,"text":"Extract columns","colour":"e6e63c"},"41203":{"id":41203,"x":967,"y":86,"width":309,"height":147,"text":"Find last change for each primary key, only if it is a delete","colour":"e6e63c"},"41206":{"id":41206,"x":1927,"y":87,"width":149,"height":148,"text":"Apply deletes","colour":"e6e63c"},"41207":{"id":41207,"x":164,"y":-154,"width":547,"height":149,"text":"**Transformation type = copy table with soft deletes**\n\nSecond transformation job for this transformation type\n\nExtract latest change events for source table, only keeping deletes and only where they are the last change per primary key. Mark the records as deleted in the target table","colour":"00ce4f"},"41208":{"id":41208,"x":1367,"y":86,"width":466,"height":226,"text":"Remove any change events where, for a given key, the MTLN_CDC_SEQUENCE_NUMBER is older than the current value","colour":"e6e63c"},"44665":{"id":44665,"x":326,"y":247,"width":148,"height":145,"text":"Using a SQL component as the Filter component does not support the nested filtering we require on the __DATE_HOUR__ and __FILENAME__ columns.","colour":"f9c21b"}},"noteConnectors":{},"variables":{"warehouse":{"definition":{"name":"warehouse","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"stage_table":{"definition":{"name":"stage_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"min_filename":{"definition":{"name":"min_filename","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"source_table":{"definition":{"name":"source_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"stage_schema":{"definition":{"name":"stage_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_table":{"definition":{"name":"target_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"min_date_hour":{"definition":{"name":"min_date_hour","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"source_schema":{"definition":{"name":"source_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_schema":{"definition":{"name":"target_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"source_version":{"definition":{"name":"source_version","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"join_expression":{"definition":{"name":"join_expression","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"source_database":{"definition":{"name":"source_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_database":{"definition":{"name":"target_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"logged_rows_deleted":{"definition":{"name":"logged_rows_deleted","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"},"logged_rows_updated":{"definition":{"name":"logged_rows_updated","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"},"logged_rows_inserted":{"definition":{"name":"logged_rows_inserted","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"}},"grids":{"join_output":{"definition":{"name":"join_output","scope":"TASKBATCH","definitions":[{"name":"input_column","type":"TEXT"},{"name":"output_column","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]},"primary_key":{"definition":{"name":"primary_key","scope":"TASKBATCH","definitions":[{"name":"key_column","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]},"delete_mapping":{"definition":{"name":"delete_mapping","scope":"TASKBATCH","definitions":[{"name":"column_name","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]},"target_metadata":{"definition":{"name":"target_metadata","scope":"TASKBATCH","definitions":[{"name":"column_name","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]},"calculated_columns":{"definition":{"name":"calculated_columns","scope":"BRANCH","definitions":[{"name":"calculation","type":"TEXT"},{"name":"alias","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]}}},"Sync Single Table - 1-2-5-2 - Transform and load":{"id":-2,"revision":1,"created":1728917529954,"timestamp":1728917529954,"components":{"41277":{"id":41277,"inputCardinality":"ZERO","outputCardinality":"MANY","implementationID":1354890871,"x":0,"y":160,"width":32,"height":32,"inputConnectorIDs":[],"outputConnectorIDs":[47196],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Stage table"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_table}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Column Names","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DATABASE"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"SCHEMA"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"TABLE"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"DATE_HOUR"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"FILENAME"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"VALUE"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"VERSION"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"FILEPATH"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Offset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"41279":{"id":41279,"inputCardinality":"ONE","outputCardinality":"MANY","implementationID":2139809017,"x":800,"y":160,"width":32,"height":32,"inputConnectorIDs":[44648],"outputConnectorIDs":[41348],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Order changes for each key"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Include input columns.","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Yes"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Partition Data","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"primary_key"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"key_column"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Ordering within partitions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_SEQUENCE_NUMBER"},"2":{"slot":2,"type":"STRING","value":"Desc"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Functions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Row Number"},"2":{"slot":2,"type":"STRING","value":"MTLN_CDC_TEMP_KEY_ROW_NUM"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"41280":{"id":41280,"inputCardinality":"ONE","outputCardinality":"MANY","implementationID":-1760161015,"x":960,"y":160,"width":32,"height":32,"inputConnectorIDs":[41348],"outputConnectorIDs":[41298],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Filter for last change only"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Filter Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_TEMP_KEY_ROW_NUM"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"AND"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"41282":{"id":41282,"inputCardinality":"ONE","outputCardinality":"MANY","implementationID":-1760161015,"x":1760,"y":80,"width":32,"height":32,"inputConnectorIDs":[44681],"outputConnectorIDs":[41297],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Filter on inserts and updates"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Filter Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_LAST_CHANGE_TYPE"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"c"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_LAST_CHANGE_TYPE"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"r"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_LAST_CHANGE_TYPE"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"u"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"OR"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"41283":{"id":41283,"inputCardinality":"ONE","outputCardinality":"MANY","implementationID":-1760161015,"x":1760,"y":240,"width":32,"height":32,"inputConnectorIDs":[44684],"outputConnectorIDs":[41300],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Filter on deletes"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Filter Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_CDC_LAST_CHANGE_TYPE"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"d"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"AND"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"41284":{"id":41284,"inputCardinality":"ONE","outputCardinality":"ZERO","implementationID":-1848704949,"x":1920,"y":240,"width":32,"height":32,"inputConnectorIDs":[41300],"outputConnectorIDs":[],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Apply deletes"}}}},"visible":true,"status":{},"columnNames":[]},"200":{"slot":200,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table}"}}}},"visible":true,"status":{},"columnNames":[]},"201":{"slot":201,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"202":{"slot":202,"name":"Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"203":{"slot":203,"name":"Warehouse","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${warehouse}"}}}},"visible":true,"status":{},"columnNames":[]},"204":{"slot":204,"name":"Target Alias","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"TARGET"}}}},"visible":true,"status":{},"columnNames":[]},"205":{"slot":205,"name":"Source Alias","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"SOURCE"}}}},"visible":true,"status":{},"columnNames":[]},"206":{"slot":206,"name":"Join Expression","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${join_expression}"},"2":{"slot":2,"type":"STRING","value":"Join"}}}},"visible":true,"status":{},"columnNames":[]},"207":{"slot":207,"name":"When Matched","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${join_expression}"},"2":{"slot":2,"type":"STRING","value":"Delete"}}}},"visible":true,"status":{},"columnNames":[]},"208":{"slot":208,"name":"Update Mapping","elements":{},"visible":false,"status":{},"columnNames":[]},"209":{"slot":209,"name":"Include Not Matched","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"No"}}}},"visible":true,"status":{},"columnNames":[]},"210":{"slot":210,"name":"Insert Mapping","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"alias"},"2":{"slot":2,"type":"GRID","value":"alias"}}}},"visible":false,"status":{},"columnNames":[]}},"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"Rows Deleted","mapTo":"logged_rows_deleted","validationStatus":null,"validationMessage":null}},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"41285":{"id":41285,"inputCardinality":"ONE","outputCardinality":"ZERO","implementationID":-1848704949,"x":1920,"y":80,"width":32,"height":32,"inputConnectorIDs":[41297],"outputConnectorIDs":[],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Apply inserts and updates"}}}},"visible":true,"status":{},"columnNames":[]},"200":{"slot":200,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table}"}}}},"visible":true,"status":{},"columnNames":[]},"201":{"slot":201,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"202":{"slot":202,"name":"Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"203":{"slot":203,"name":"Warehouse","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${warehouse}"}}}},"visible":true,"status":{},"columnNames":[]},"204":{"slot":204,"name":"Target Alias","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"TARGET"}}}},"visible":true,"status":{},"columnNames":[]},"205":{"slot":205,"name":"Source Alias","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"SOURCE"}}}},"visible":true,"status":{},"columnNames":[]},"206":{"slot":206,"name":"Join Expression","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${join_expression}"},"2":{"slot":2,"type":"STRING","value":"Join"}}}},"visible":true,"status":{},"columnNames":[]},"207":{"slot":207,"name":"When Matched","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${join_expression}"},"2":{"slot":2,"type":"STRING","value":"Update"}}}},"visible":true,"status":{},"columnNames":[]},"208":{"slot":208,"name":"Update Mapping","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"target_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"column_name"},"2":{"slot":2,"type":"GRID","value":"column_name"}}}},"visible":true,"status":{},"columnNames":[]},"209":{"slot":209,"name":"Include Not Matched","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Yes"}}}},"visible":true,"status":{},"columnNames":[]},"210":{"slot":210,"name":"Insert Mapping","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"target_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"column_name"},"2":{"slot":2,"type":"GRID","value":"column_name"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"Rows Inserted","mapTo":"logged_rows_inserted","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"Rows Updated","mapTo":"logged_rows_updated","validationStatus":null,"validationMessage":null}},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"41286":{"id":41286,"inputCardinality":"ONE","outputCardinality":"MANY","implementationID":1716658327,"x":400,"y":160,"width":32,"height":32,"inputConnectorIDs":[47232],"outputConnectorIDs":[44642],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Extract payload and metadata objects"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Include Input Columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"No"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Calculations","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"CASE \"VALUE\":\"metadata\".\"op\"\n WHEN 'd' THEN \"VALUE\":\"before\"\n ELSE \"VALUE\":\"after\"\nEND"},"2":{"slot":2,"type":"STRING","value":"MTLN_CDC_EXT_PAYLOAD"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"\"VALUE\":\"metadata\""},"2":{"slot":2,"type":"STRING","value":"MTLN_CDC_EXT_METADATA"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"\"VERSION\""},"2":{"slot":2,"type":"STRING","value":"MTLN_CDC_EXT_VERSION"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"\"DATE_HOUR\""},"2":{"slot":2,"type":"STRING","value":"MTLN_CDC_EXT_DATE_HOUR"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"\"FILENAME\""},"2":{"slot":2,"type":"STRING","value":"MTLN_CDC_EXT_FILENAME"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"\"FILEPATH\""},"2":{"slot":2,"type":"STRING","value":"MTLN_CDC_EXT_FILEPATH"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"41288":{"id":41288,"inputCardinality":"ZERO","outputCardinality":"MANY","implementationID":1354890871,"x":1200,"y":240,"width":32,"height":32,"inputConnectorIDs":[],"outputConnectorIDs":[41344],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Target table"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Column Names","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"target_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"column_name"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Offset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"41289":{"id":41289,"inputCardinality":"MANY","outputCardinality":"MANY","implementationID":-629958239,"x":1360,"y":160,"width":32,"height":32,"inputConnectorIDs":[41344,41298],"outputConnectorIDs":[41296],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Join existing data"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Main Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Filter for last change only"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Main Table Alias","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"SOURCE"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Joins","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Target table"},"2":{"slot":2,"type":"STRING","value":"TARGET"},"3":{"slot":3,"type":"STRING","value":"Left"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Join Expressions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${join_expression}"},"2":{"slot":2,"type":"STRING","value":"SOURCE_Left_TARGET"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Output Columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"join_output"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"input_column"},"2":{"slot":2,"type":"GRID","value":"output_column"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"41290":{"id":41290,"inputCardinality":"ONE","outputCardinality":"MANY","implementationID":-1760161015,"x":1520,"y":160,"width":32,"height":32,"inputConnectorIDs":[41296],"outputConnectorIDs":[44681,44684],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Filter for new data with newer MTLN_CDC_SEQUENCE_NUMBER"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Filter Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_TARGET_KEY"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Null"},"4":{"slot":4,"type":"STRING","value":""}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"MTLN_TARGET_MTLN_CDC_SEQUENCE_NUMBER"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Less than"},"4":{"slot":4,"type":"STRING","value":"MTLN_CDC_SEQUENCE_NUMBER"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"OR"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"44634":{"id":44634,"inputCardinality":"ONE","outputCardinality":"MANY","implementationID":1716658327,"x":560,"y":160,"width":32,"height":32,"inputConnectorIDs":[44642],"outputConnectorIDs":[44648],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Extract nested columns"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Include Input Columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"No"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Calculations","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"calculated_columns"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"calculation"},"2":{"slot":2,"type":"GRID","value":"alias"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"47188":{"id":47188,"inputCardinality":"ZERO_OR_MANY","outputCardinality":"MANY","implementationID":-1266674941,"x":160,"y":160,"width":32,"height":32,"inputConnectorIDs":[47196],"outputConnectorIDs":[47232],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Filter on source table"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"SQL Query","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"SELECT \n * \nFROM $T{Stage table}\nWHERE \n \"DATABASE\" = '${source_database}'\n AND \"SCHEMA\" = '${source_schema}'\n AND \"TABLE\" = '${source_table}'\n AND \"VERSION\" = ${source_version}\n AND (\n (\n \"DATE_HOUR\" = '${min_date_hour}'\n AND \"FILENAME\" >= '${min_filename}'\n )\n OR\n (\n \"DATE_HOUR\" > '${min_date_hour}'\n )\n )"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"}},"connectors":{"41296":{"id":41296,"sourceID":41289,"targetID":41290},"41297":{"id":41297,"sourceID":41282,"targetID":41285},"41298":{"id":41298,"sourceID":41280,"targetID":41289},"41300":{"id":41300,"sourceID":41283,"targetID":41284},"41344":{"id":41344,"sourceID":41288,"targetID":41289},"41348":{"id":41348,"sourceID":41279,"targetID":41280},"44642":{"id":44642,"sourceID":41286,"targetID":44634},"44648":{"id":44648,"sourceID":44634,"targetID":41279},"44681":{"id":44681,"sourceID":41290,"targetID":41282},"44684":{"id":44684,"sourceID":41290,"targetID":41283},"47196":{"id":47196,"sourceID":41277,"targetID":47188},"47232":{"id":47232,"sourceID":47188,"targetID":41286}},"canUndo":false,"undoCommand":"","undoCreated":0,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"41268":{"id":41268,"x":-72,"y":87,"width":308,"height":147,"text":"Read data from stage table, filter for source table and files newer than __max_filename__","colour":"e6e63c"},"41269":{"id":41269,"x":327,"y":87,"width":308,"height":148,"text":"Extract columns","colour":"e6e63c"},"41270":{"id":41270,"x":725,"y":86,"width":309,"height":147,"text":"Find last change for each primary key","colour":"e6e63c"},"41271":{"id":41271,"x":1687,"y":7,"width":306,"height":144,"text":"Apply insert and updates","colour":"e6e63c"},"41272":{"id":41272,"x":1687,"y":168,"width":307,"height":145,"text":"Apply deletes","colour":"e6e63c"},"41275":{"id":41275,"x":-73,"y":-153,"width":547,"height":149,"text":"**Transformation type = copy table**\n\nExtract latest change events for source table, and apply the last change per primary key value to the target table","colour":"00ce4f"},"41276":{"id":41276,"x":1125,"y":87,"width":466,"height":226,"text":"Remove any change events where, for a given key, the MTLN_CDC_SEQUENCE_NUMBER is older than the current value","colour":"e6e63c"},"44629":{"id":44629,"x":86,"y":248,"width":148,"height":145,"text":"Using a SQL component as the Filter component does not support the nested filtering we require on the __DATE_HOUR__ and __FILENAME__ columns.","colour":"f9c21b"}},"noteConnectors":{},"variables":{"warehouse":{"definition":{"name":"warehouse","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"stage_table":{"definition":{"name":"stage_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"min_filename":{"definition":{"name":"min_filename","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"source_table":{"definition":{"name":"source_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"stage_schema":{"definition":{"name":"stage_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_table":{"definition":{"name":"target_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"min_date_hour":{"definition":{"name":"min_date_hour","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"source_schema":{"definition":{"name":"source_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_schema":{"definition":{"name":"target_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"source_version":{"definition":{"name":"source_version","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"join_expression":{"definition":{"name":"join_expression","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"source_database":{"definition":{"name":"source_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_database":{"definition":{"name":"target_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"nested_select_list":{"definition":{"name":"nested_select_list","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"logged_rows_deleted":{"definition":{"name":"logged_rows_deleted","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"},"logged_rows_updated":{"definition":{"name":"logged_rows_updated","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"},"logged_rows_inserted":{"definition":{"name":"logged_rows_inserted","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"}},"grids":{"join_output":{"definition":{"name":"join_output","scope":"TASKBATCH","definitions":[{"name":"input_column","type":"TEXT"},{"name":"output_column","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]},"primary_key":{"definition":{"name":"primary_key","scope":"TASKBATCH","definitions":[{"name":"key_column","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]},"target_metadata":{"definition":{"name":"target_metadata","scope":"TASKBATCH","definitions":[{"name":"column_name","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]},"calculated_columns":{"definition":{"name":"calculated_columns","scope":"TASKBATCH","definitions":[{"name":"calculation","type":"TEXT"},{"name":"alias","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]}}}}},{"metadata":{"identifier":{"packageName":"Matillion.Data Loader.CDC","name":"Drop CDC Tables (Exchange)","revision":1,"type":"DYNAMIC"},"rootJobReference":{"name":"Drop CDC Tables - 1 - Iterate over tables","type":"ORCHESTRATION","parameterMetadata":[{"slot":2,"variableName":"cloud_storage_url","variableType":"SCALAR","displayName":"Cloud Storage URL","description":"The URL of the location where the CDC change event files are stored. This should have the format:\n\ns3:////\nor\nazure://.blob.core.windows.net///\nor\ngs:////","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":3,"variableName":"target_database","variableType":"SCALAR","displayName":"Target Database","description":"The Snowflake database containing the external table with change events and the target table.","defaultValue":[{"values":{"1":"[Environment Default]"}}],"defaultValueType":"TEXT","required":true},{"slot":4,"variableName":"target_schema","variableType":"SCALAR","displayName":"Target Schema","description":"The Snowflake schema containing the external table with change events and the target table.","defaultValue":[{"values":{"1":"[Environment Default]"}}],"defaultValueType":"TEXT","required":true},{"slot":5,"variableName":"external_table","variableType":"SCALAR","displayName":"External Table","description":"The external table in Snowflake containing the CDC change events.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":6,"variableName":"target_prefix","variableType":"SCALAR","displayName":"Target Table Prefix","description":"A prefix to prepend to the source table name to generate the target table name. If no Target Table Prefix is specified, the target table will have the same name as the source table.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":false},{"slot":7,"variableName":"fully_qualify_target_table","variableType":"SCALAR","displayName":"Fully Qualify Target Table","description":"Optionally includes the source database and schema in the target table name.\n\nY: The target table name will have the format \n__\n\nN: The target table name will have the format \n\n\nIt is strongly recommended to leave this set to 'Y', unless you are confident that your source table names will always be unique.\n\nIf the target table name is not fully qualified, and there are source tables with the same name in different databases or schemas, this job will see them as the same table and attempt to merge their metadata and contents.","defaultValue":[{"values":{"1":"Y"}}],"defaultValueType":"TEXT","required":true},{"slot":8,"variableName":"azure_key_vault_url","variableType":"SCALAR","displayName":"Azure Key Vault URL","description":"Required for Azure Blob Storage\nThe URL of the Azure Key Vault used to store the Azure Blob Storage connection string. This is also called the vault's \"DNS Name\".","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":false},{"slot":9,"variableName":"storage_account_key_secret","variableType":"SCALAR","displayName":"Azure Blob Storage Account Key Secret","description":"Required for Azure Blob Storage\nName of the Azure Key Vault secret which stores the Azure Blob Storage account key.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":false},{"slot":10,"variableName":"drop_tables","variableType":"SCALAR","displayName":"Actually drop the tables","description":"Optionally drop the CDC tables from the target database.\n\nY: The CDC tables will be dropped.\n\nN: The CDC tables will not be dropped. A list of the tables can be exported from the cdc_tables grid variable.","defaultValue":[{"values":{"1":"N"}}],"defaultValueType":"TEXT","required":true},{"slot":11,"variableName":"cdc_tables","variableType":"GRID","displayName":"CDC Tables (for exporting only)","description":"Do not set this parameter, it will have no effect. A cdc_tables grid variable can be exported, containing the full list of tables created by Sync All Tables shared job.","defaultValue":[{"values":{"1":""}}],"defaultValueType":null,"required":false}]},"orchestrationJobs":["Drop CDC Tables - 1 - Iterate over tables","Drop CDC Tables - 1-1 - Get source tables"],"transformationJobs":[],"imageMetadata":{"componentIcon":"iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAABaUlEQVR42mNgGAVYgJKSUj0Q/wbi/3jwb2Vl5RpaOQBk+WUgnoAD/0dyYD0tHPBfUVFxAQH5FUC8nSaOINIBCxQUFDho4ghiHQBiOzg4sADZq0FiwDTRTXcHwBwBFFsODYl+ejjgPRSfR8PghEkPB/gAg3sHEB9AxkDxx9BQAOcWoFgETRyAR98ELOWFD90dAMwdBkAcAXVAAd0dAGKDHDHqgFEHjDwHAPXkgAokEBtIq4DYQLEQujmAbrUhNhA6X0EiYL6CQuAyhQlAfB6B5VeDxEGYpg4AWnYBiP/jw6Q4AFTTfQbi+4QwqGUE0uO3SMkkYLFCAC7sv1TJhWgHAFOwA9DgNcBEdByakk8D+RuwYaCaFpAeYBAL4PM90BEOJKcFoEMCoJVLAiG1NHEAqCqFNrUmA3EANgyUt6FlCCgALfiOr38Ay/c0cQCsRAMVMEC6ABuGFTQ0cwCxYLA44D1uDEkrowAdAAAjP4UCFgLNagAAAABJRU5ErkJggg==","smallIcon":"iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAA6UlEQVR42mNgoBZQVlaOV1JSug6kj4MwkP0YiO8oAAFRBgAVNwA1BiDzFRUVm4D0WaIMwWYAiA+kdYH4DJCtQpIBKioq/iDbod65DcSLgWwrog1ACx8PoPxpIL4K9JY+uQY0APEMYHBYDEED1NTUpIEaDYABawnEoiQbELBcvilwmcJ1IH4etExhE5D2wGaADxBfBOJHsNQIjULvwCWKqQFLFVbAsO8ieUWsrgBGkTpQwx6gczVgGMjnD1yu0AG09T8MByzHkTKBGjiAGpYADdqOhF2JNgAXCFym2ALU+B2Gg5YqylMtFwMAOXVvS8kAGuQAAAAASUVORK5CYII="},"description":"Drop all the tables created by the Sync All Tables and Sync Single Tables shared jobs.","helpHtml":"

Drop CDC Tables

Drop all the tables created by the Sync All Tables and Sync Single Tables shared jobs.

Properties

PropertyTypeDescription
Cloud Storage URLVariableThe URL of the location where the CDC change event files are stored. This should have the format:\n\ns3://<bucket>/<prefix>/\nor\nazure://<account>.blob.core.windows.net/<container>/<prefix>/\nor\ngs://<bucket>/<prefix>/
Target DatabaseVariableThe Snowflake database containing the external table with change events and the target table.
Target SchemaVariableThe Snowflake schema containing the external table with change events and the target table.
External TableVariableThe external table in Snowflake containing the CDC change events.
Target Table PrefixVariableA prefix to prepend to the source table name to generate the target table name. If no Target Table Prefix is specified, the target table will have the same name as the source table.
Fully Qualify Target TableVariableOptionally includes the source database and schema in the target table name.\n\nY: The target table name will have the format \n<source_database>_<source_schema>_<source_table>\n\nN: The target table name will have the format \n<source_table>\n\nIt is strongly recommended to leave this set to 'Y', unless you are confident that your source table names will always be unique.\n\nIf the target table name is not fully qualified, and there are source tables with the same name in different databases or schemas, this job will see them as the same table and attempt to merge their metadata and contents.
Azure Key Vault URLVariableRequired for Azure Blob Storage\nThe URL of the Azure Key Vault used to store the Azure Blob Storage connection string. This is also called the vault's "DNS Name".
Azure Blob Storage Account Key SecretVariableRequired for Azure Blob Storage\nName of the Azure Key Vault secret which stores the Azure Blob Storage account key.
Actually drop the tablesVariableOptionally drop the CDC tables from the target database.\n\nY: The CDC tables will be dropped.\n\nN: The CDC tables will not be dropped. A list of the tables can be exported from the cdc_tables grid variable.
CDC Tables (for exporting only)GridDo not set this parameter, it will have no effect. A cdc_tables grid variable can be exported, containing the full list of tables created by Sync All Tables shared job.
","created":1657623956407},"orchestrationJobs":{"Drop CDC Tables - 1 - Iterate over tables":{"id":-2,"revision":1,"created":1657523977345,"timestamp":1657523977345,"components":{"44309":{"id":44309,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":-1120,"y":0,"width":32,"height":32,"inputConnectorIDs":[50265],"outputSuccessConnectorIDs":[44317],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Update [Environment Default] values"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"if target_database == '[Environment Default]':\n print(f'Updating target_database from [Environment Default] to: {environment_database}')\n context.updateVariable('target_database', environment_database)\n\nif target_schema == '[Environment Default]':\n print(f'Updating target_schema from [Environment Default] to: {environment_default_schema}')\n context.updateVariable('target_schema', environment_default_schema)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44310":{"id":44310,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":-640,"y":0,"width":32,"height":32,"inputConnectorIDs":[44318],"outputSuccessConnectorIDs":[44471],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Build external table names"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"from pprint import pprint\n\n\nexternal_table = external_table.upper()\nexternal_metadata_table = f'{external_table}_VERSION_METADATA'\n\ncdc_tables = [\n [external_table],\n [external_metadata_table],\n]\n\nprint()\nprint('cdc_tables:')\npprint(cdc_tables)\ncontext.updateGridVariable('cdc_tables', cdc_tables)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44311":{"id":44311,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":-1520,"y":0,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[50260],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Start","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44313":{"id":44313,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":240,"y":0,"width":32,"height":32,"inputConnectorIDs":[44703],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Success - Tables dropped"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44314":{"id":44314,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":-880,"y":0,"width":32,"height":32,"inputConnectorIDs":[44317],"outputSuccessConnectorIDs":[44318],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"gridExportMappings":{"1":{"slot":1,"fromGrid":"source_tables","toGrid":"source_tables","columnMapToFrom":{"source_table":"source_table","source_schema":"source_schema","source_database":"source_database"},"validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Get list of source tables in pipeline"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Drop CDC Tables - 1-1 - Get source tables"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"cloud_storage_url"},"2":{"slot":2,"type":"STRING","value":"${cloud_storage_url}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"azure_key_vault_url"},"2":{"slot":2,"type":"STRING","value":"${azure_key_vault_url}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"storage_account_key_secret"},"2":{"slot":2,"type":"STRING","value":"${storage_account_key_secret}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Drop CDC Tables (Exchange)"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Data Loader.CDC"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44464":{"id":44464,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":-400,"y":0,"width":32,"height":32,"inputConnectorIDs":[44471],"outputSuccessConnectorIDs":[44671],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Build target table names"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"from pprint import pprint\n\n\nif not target_prefix:\n target_prefix = ''\n\nprint(f'fully_qualify_target_table: {fully_qualify_target_table}')\n\nsource_tables = context.getGridVariable('source_tables') or []\ncdc_tables = context.getGridVariable('cdc_tables') or []\n\nfor source_database, source_schema, source_table in source_tables:\n\n if fully_qualify_target_table.lower() in ('yes', 'y', 'true', 't', '1'):\n fully_qualified_prefix = f'{source_database}_{source_schema}_'\n else:\n fully_qualified_prefix = ''\n \n cdc_tables.append(\n [f'{target_prefix}{fully_qualified_prefix}{source_table}'.strip().upper()]\n )\n\nprint()\nprint('cdc_tables:')\npprint(cdc_tables)\ncontext.updateGridVariable('cdc_tables', cdc_tables)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44529":{"id":44529,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":-160,"y":0,"width":32,"height":32,"inputConnectorIDs":[44671],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[44537],"outputFalseConnectorIDs":[44713],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Drop tables?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Advanced"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":false,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"['yes', 'y', 'true', 't', '1'].indexOf(drop_tables.toLowerCase()) !== -1"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44536":{"id":44536,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1291941950,"x":80,"y":0,"width":32,"height":32,"inputConnectorIDs":[44537],"outputSuccessConnectorIDs":[44703],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Drop tables"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"cdc_tables"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"table"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Ignore Missing","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Yes"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44706":{"id":44706,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":-160,"y":160,"width":32,"height":32,"inputConnectorIDs":[44713],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Success - No tables dropped"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"50247":{"id":50247,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":-1360,"y":0,"width":32,"height":32,"inputConnectorIDs":[50260],"outputSuccessConnectorIDs":[50265],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Print version"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"print('Version: 1.0.0')"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"44317":{"id":44317,"sourceID":44309,"targetID":44314},"44318":{"id":44318,"sourceID":44314,"targetID":44310},"44471":{"id":44471,"sourceID":44310,"targetID":44464},"44671":{"id":44671,"sourceID":44464,"targetID":44529},"44703":{"id":44703,"sourceID":44536,"targetID":44313},"50265":{"id":50265,"sourceID":50247,"targetID":44309}},"failureConnectors":{},"unconditionalConnectors":{"50260":{"id":50260,"sourceID":44311,"targetID":50247}},"trueConnectors":{"44537":{"id":44537,"sourceID":44529,"targetID":44536}},"falseConnectors":{"44713":{"id":44713,"sourceID":44529,"targetID":44706}},"iterationConnectors":{},"noteConnectors":{},"canUndo":false,"undoCommand":"","undoCreated":0,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"44286":{"id":44286,"x":-475,"y":-153,"width":150,"height":229,"text":"**Build target table names**\n\nBuild __target_table__ from __target_prefix__, __source_database__, __source_schema__and __source_table__\n","colour":"e6e63c"},"44287":{"id":44287,"x":-1195,"y":-154,"width":150,"height":228,"text":"**Update [Environment Default] values**\n\nReplace any [Environment Default] parameters with the actual value","colour":"e6e63c"},"44301":{"id":44301,"x":-2234,"y":-393,"width":631,"height":147,"text":"**Build a list of all tables in CDC pipeline, so that they can be dropped**\n","colour":"00ce4f"},"44302":{"id":44302,"x":-715,"y":-154,"width":151,"height":228,"text":"**Build external table names**\n\nEnsure external table name is upper case.","colour":"e6e63c"},"44303":{"id":44303,"x":5,"y":-155,"width":149,"height":229,"text":"**Drop tables**","colour":"e6e63c"},"44304":{"id":44304,"x":-2234,"y":-154,"width":626,"height":389,"text":"**Change log**\n\n1.0.0 -----------------------------------------------------------------------------------------------------------\n- Initial version\n","colour":"00ce4f"},"44305":{"id":44305,"x":-955,"y":-155,"width":150,"height":230,"text":"**Get list of source tables**\n\nQuery cloud storage to obtain a list of all source tables in this CDC pipeline","colour":"e6e63c"},"44306":{"id":44306,"x":-234,"y":-155,"width":148,"height":230,"text":"**Check drop_tables job variable**","colour":"e6e63c"},"50248":{"id":50248,"x":-1437,"y":-155,"width":152,"height":229,"text":"**Print version**\n\nPrint shared job version to the task history, to help support cases","colour":"e6e63c"}},"variables":{"drop_tables":{"definition":{"name":"drop_tables","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"N"},"target_prefix":{"definition":{"name":"target_prefix","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_schema":{"definition":{"name":"target_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"external_table":{"definition":{"name":"external_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_database":{"definition":{"name":"target_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"cloud_storage_url":{"definition":{"name":"cloud_storage_url","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"azure_key_vault_url":{"definition":{"name":"azure_key_vault_url","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"fully_qualify_target_table":{"definition":{"name":"fully_qualify_target_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"Y"},"storage_account_key_secret":{"definition":{"name":"storage_account_key_secret","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""}},"grids":{"cdc_tables":{"definition":{"name":"cdc_tables","scope":"TASKBATCH","definitions":[{"name":"table","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]},"source_tables":{"definition":{"name":"source_tables","scope":"TASKBATCH","definitions":[{"name":"source_database","type":"TEXT"},{"name":"source_schema","type":"TEXT"},{"name":"source_table","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[]}}},"Drop CDC Tables - 1-1 - Get source tables":{"id":-2,"revision":1,"created":1657523977261,"timestamp":1657523977261,"components":{"44328":{"id":44328,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":720,"y":240,"width":32,"height":32,"inputConnectorIDs":[44349],"outputSuccessConnectorIDs":[44348],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Get list of tables from GCS"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"\"\"\"Get a list of source tables from Google Cloud Storage.\"\"\"\nfrom pprint import pprint\nfrom typing import List\nfrom typing import Set\n\nfrom google.cloud import storage # type: ignore\n\nfrom . import *\n\n\ndef get_source_table_paths(bucket: str, prefix: str) -> List[str]:\n \"\"\"Iterate over database and schema folders to get a list of table paths.\n\n Args:\n bucket: Google Cloud Storage bucket\n prefix: Prefix for object paths in bucket\n\n Returns:\n List[str]: List of strings of format 'database=/schema=/table=/'\n \"\"\"\n client = storage.Client()\n\n source_table_paths: List[str] = []\n database_paths = get_child_folders(client, bucket, prefix)\n\n for database in database_paths:\n schema_paths = get_child_folders(client, bucket, database)\n\n for schema in schema_paths:\n source_table_paths.extend(get_child_folders(client, bucket, schema))\n\n return source_table_paths\n\n\ndef get_child_folders(\n client: storage.client.Client, bucket: str, prefix: str\n) -> Set[str]:\n \"\"\"Get list of child folders under '/'.\n\n Args:\n client: Google storage client\n bucket: Google Cloud Storage bucket\n prefix: Prefix for object paths in bucket\n\n Returns:\n Set[str]: List of strings of format '/'\n \"\"\"\n blob_iterator = client.list_blobs(\n bucket_or_name=bucket,\n prefix=prefix,\n delimiter='/',\n include_trailing_delimiter=True,\n )\n\n # exhaust generator, without storing values\n # blob_iterator will build a set of prefixes as it yields items\n for _ in blob_iterator:\n pass\n\n return blob_iterator.prefixes # type: ignore\n\n\ndef extract_source_tables(paths: List[str], prefix: str) -> List[List[str]]:\n \"\"\"Convert list of table paths into lists of [, ,
].\n\n Args:\n paths: List of strings of format 'database=/schema=/table=
/'\n prefix: Prefix for object paths in bucket\n\n Returns:\n List[List[str]]: List of lists of format [, ,
]\n \"\"\"\n source_tables = []\n for path in paths:\n try:\n source_tables.append(split_table_path(path, prefix))\n except IndexError:\n # ignore paths which do not contain '='\n pass\n\n return source_tables\n\n\ndef split_table_path(path: str, prefix: str) -> List[str]:\n \"\"\"Split table path into list of [, ,
].\n\n Args:\n path: String of format 'database=/schema=/table=
/'\n prefix: Prefix for object paths in bucket\n\n Returns:\n List[str]: List of strings of format [, ,
]\n\n Raises:\n IndexError: If any folder does not contain '='\n \"\"\"\n key_value_list = path[len(prefix) : -1].split('/')\n\n value_list = []\n for key_value in key_value_list:\n try:\n value_list.append(key_value.split('=', 1)[1])\n except IndexError:\n raise\n\n return value_list\n\n\ndef split_host_path(host_path: str) -> List[str]:\n \"\"\"Split host_path into host and path.\n\n Args:\n host_path: String of format '//'\n\n Returns:\n List[str]: List of format ['', '/']\n \"\"\"\n return host_path.split('/', 1)\n\n\ndef main() -> None:\n \"\"\"Get a list of source tables from Google Cloud Storage.\"\"\"\n bucket, prefix = split_host_path(host_path)\n print(f'bucket: {bucket!r}')\n print(f'prefix: {prefix!r}')\n print()\n\n # get a list of paths of form 'database=/schema=/table=
/'\n source_table_paths = get_source_table_paths(bucket, prefix)\n\n # convert to list of lists of form '[, ,
]'\n source_tables = extract_source_tables(source_table_paths, prefix)\n\n print('source_tables:')\n pprint(source_tables)\n context.updateGridVariable('source_tables', source_tables)\n\n\nif __name__ == \"__main__\":\n main()\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44329":{"id":44329,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":560,"y":320,"width":32,"height":32,"inputConnectorIDs":[44352],"outputSuccessConnectorIDs":[44343],"outputFailureConnectorIDs":[44358],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Validate Azure parameters"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"print(f'azure_key_vault_url: {azure_key_vault_url!r}')\nprint(f'storage_account_key_secret: {storage_account_key_secret!r}')\n\nmissing_parameters = []\n\nif not azure_key_vault_url:\n missing_parameters.append('Azure Key Vault URL')\n \nif not storage_account_key_secret:\n missing_parameters.append('Azure Blob Storage Account Key Secret')\n \nif missing_parameters:\n err_msg = ('The following shared job parameters were not set, but are '\n f'required for Azure Blob Storage: {missing_parameters}')\n raise Exception(err_msg)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44330":{"id":44330,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":560,"y":480,"width":32,"height":32,"inputConnectorIDs":[44358],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Failure - Azure Blob Storage parameters not set"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44331":{"id":44331,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":74,"y":155,"width":32,"height":32,"inputConnectorIDs":[44356],"outputSuccessConnectorIDs":[44344],"outputFailureConnectorIDs":[44357],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Identify storage protocol"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"if not cloud_storage_url[-1:] == '/':\n cloud_storage_url = cloud_storage_url + '/'\n \nprint(f'cloud_storage_url: {cloud_storage_url!r}')\nprint()\n\n# split the cloud_storage_url into protocol and host_path\nurl_list = cloud_storage_url.split('://', 1)\n\n# check we could split the string and extract a protocol\nif len(url_list) == 1:\n err_msg = f'Unable to determine the protocol from the cloud_storage_url {cloud_storage_url!r}'\n raise Exception(err_msg)\n\n# unpack the list\nprotocol, host_path = url_list\nprotocol = protocol.lower()\n\n# check protocol is supported\nsupported_protocols = ('azure', 's3', 'gs', 'gcs')\nif protocol not in supported_protocols:\n err_msg = f'The protocol {protocol!r} was identifed from the cloud_storage_url {cloud_storage_url!r}, but the only supported protocols are {\", \".join(supported_protocols)}'\n raise Exception(err_msg)\n\nprint(f'protocol: {protocol!r}')\ncontext.updateVariable('protocol', protocol)\n\nprint(f'host_path: {host_path!r}')\ncontext.updateVariable('host_path', host_path)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44332":{"id":44332,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":80,"y":480,"width":32,"height":32,"inputConnectorIDs":[44357],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Failure - Storage protocol not supported"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44333":{"id":44333,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":234,"y":155,"width":32,"height":32,"inputConnectorIDs":[44344],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[44351],"outputFalseConnectorIDs":[44353],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Protocol is s3?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"protocol"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"s3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44337":{"id":44337,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":720,"y":160,"width":32,"height":32,"inputConnectorIDs":[44351],"outputSuccessConnectorIDs":[44347],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Get list of tables from S3"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"import boto3\nfrom pprint import pprint\n\n\ndef get_common_prefixes(client, bucket, prefix, continuation_token=None):\n \"\"\"\n Wrapper for list_objects_v2 which returns all items when more than 1000\n \n Returns a list of dicts like [{'Prefix': 'niagara-shared-job-test/stage/postgres/'}]\n \"\"\"\n if continuation_token:\n response = client.list_objects_v2(\n Bucket=bucket, \n Prefix=prefix, \n Delimiter='/',\n ContinuationToken=continuation_token,\n )\n else:\n response = client.list_objects_v2(\n Bucket=bucket, \n Prefix=prefix, \n Delimiter='/',\n )\n \n common_prefixes = response.get('CommonPrefixes', list())\n \n if response.get('IsTruncated'):\n continuation_token = response.get('NextContinuationToken')\n common_prefixes.extend(\n get_common_prefixes(client, bucket, prefix, continuation_token)\n )\n \n return common_prefixes\n\n\ndef get_sub_paths(client, bucket, prefix):\n \"\"\"\n Extract list of paths from common_prefixes list\n \"\"\"\n common_prefixes = get_common_prefixes(client, bucket, prefix)\n sub_paths = [c.get('Prefix') for c in common_prefixes]\n return sub_paths\n\n\ndef split_bucket_prefix(s):\n \"\"\"\n Take base_path and return list of [bucket, path]\n \"\"\"\n path_list = s.split('/', 1)\n if len(path_list) == 1:\n return path_list.append('')\n else:\n return path_list\n\n\ndef split_path(s, prefix):\n \"\"\"\n Split path into list of [, ,
]\n \n Input path has format 'database=/schema=/table=
/'\n \"\"\"\n key_value_list = s[len(prefix):-1].split('/')\n return [key_value.split('=', 1)[1] for key_value in key_value_list]\n\n\nclient = boto3.client('s3')\n\n# split stage_base into bucket and prefix\nbucket, prefix = split_bucket_prefix(host_path)\nprint(f'bucket: {bucket!r}')\nprint(f'prefix: {prefix!r}')\n\n# get a list of paths of form '///
'\nsource_table_paths = []\ndatabase_paths = get_sub_paths(client, bucket, prefix)\nfor database in database_paths:\n schema_paths = get_sub_paths(client, bucket, database)\n for schema in schema_paths:\n source_table_paths.extend(get_sub_paths(client, bucket, schema))\n\n# convert to list of lists before storing as grid var, of form '[, ,
]'\nsource_tables = [\n split_path(path, prefix) for path in source_table_paths\n]\n\nprint('source_tables:')\npprint(source_tables)\ncontext.updateGridVariable('source_tables', source_tables)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44338":{"id":44338,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":720,"y":320,"width":32,"height":32,"inputConnectorIDs":[44343],"outputSuccessConnectorIDs":[44346],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Get list of tables from Azure"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"try:\n from azure.identity import ManagedIdentityCredential\n from azure.keyvault.secrets import SecretClient\n from azure.storage.blob import BlobServiceClient\nexcept ModuleNotFoundError as e:\n err_msg = ('Module not found. The following Python3 packages are '\n 'required: [azure-identity, azure-keyvault-secrets, azure-storage-blob]'\n f'\\nException: {e!r}')\n raise ModuleNotFoundError(err_msg)\n \nfrom collections import namedtuple\nfrom pprint import pprint \n\n\n# create a named tuple so that we can use sets to get distinct list of tables\nTableReference = namedtuple('TableReference', 'database schema table')\n\n\ndef get_distinct_tables(blobs, prefix):\n \"\"\"\n Extract a list of distinct ['database', 'schema', 'table'] from a list of blobs\n \"\"\"\n all_table_refs = []\n for blob in blobs:\n \n # get blob name without 'prefix'\n blob_name = blob.name[len(prefix):]\n \n # try to split blob_name into ['database=', 'schema=', 'table=
', '....']\n blob_path_list = blob_name.split('/', 3)\n if len(blob_path_list) == 4:\n table_list = [key_value.split('=', 1)[1] for key_value in blob_path_list[:-1]]\n all_table_refs.append(\n TableReference(*table_list)\n )\n \n distinct_table_refs = set(all_table_refs)\n \n # convert named tuples to lists before returning\n return [list(t) for t in distinct_table_refs]\n \n\n# split host_path into account, container and prefix\naccount, container, prefix = host_path.split('/', 2)\nprint(f'account: {account!r}')\nprint(f'container: {container!r}')\nprint(f'prefix: {prefix!r}')\nprint()\n\n# lookup storage account key from Azure Key Vault\nprint(f'Attempting to read the storage account key from the secret {storage_account_key_secret!r} in the Key Vault {azure_key_vault_url!r}')\ncredential = ManagedIdentityCredential()\nkey_vault_client = SecretClient(vault_url=azure_key_vault_url, credential=credential)\nkey_vault_secret = key_vault_client.get_secret(storage_account_key_secret)\n\naccount_key = key_vault_secret.value\nprint('Successfully read storage account key')\nprint()\n\n# get list of blobs in container which start with prefix\nblob_service_client = BlobServiceClient(\n account_url=f'https://{account}',\n credential=account_key\n)\ncontainer_client = blob_service_client.get_container_client(container)\nblobs = container_client.list_blobs(name_starts_with=prefix)\n\n# get distinct list of source tables from blob names\nsource_tables = get_distinct_tables(blobs, prefix)\n\nprint('source_tables:')\npprint(source_tables)\ncontext.updateGridVariable('source_tables', source_tables)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44339":{"id":44339,"inputCardinality":"MANY","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1343684451,"x":880,"y":160,"width":32,"height":32,"inputConnectorIDs":[44346,44347,44348],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[44435],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Join paths"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44340":{"id":44340,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":-80,"y":160,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[44356],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44341":{"id":44341,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":1040,"y":160,"width":32,"height":32,"inputConnectorIDs":[44435],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Success"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"44342":{"id":44342,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":384,"y":240,"width":32,"height":32,"inputConnectorIDs":[44353],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[44349],"outputFalseConnectorIDs":[44352],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Protocol is gs?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"protocol"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"gs"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"protocol"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"gcs"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Or"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"44343":{"id":44343,"sourceID":44329,"targetID":44338},"44344":{"id":44344,"sourceID":44331,"targetID":44333},"44346":{"id":44346,"sourceID":44338,"targetID":44339},"44347":{"id":44347,"sourceID":44337,"targetID":44339},"44348":{"id":44348,"sourceID":44328,"targetID":44339}},"failureConnectors":{"44357":{"id":44357,"sourceID":44331,"targetID":44332},"44358":{"id":44358,"sourceID":44329,"targetID":44330}},"unconditionalConnectors":{"44356":{"id":44356,"sourceID":44340,"targetID":44331},"44435":{"id":44435,"sourceID":44339,"targetID":44341}},"trueConnectors":{"44349":{"id":44349,"sourceID":44342,"targetID":44328},"44351":{"id":44351,"sourceID":44333,"targetID":44337}},"falseConnectors":{"44352":{"id":44352,"sourceID":44342,"targetID":44329},"44353":{"id":44353,"sourceID":44333,"targetID":44342}},"iterationConnectors":{},"noteConnectors":{},"canUndo":false,"undoCommand":"","undoCreated":0,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"44327":{"id":44327,"x":6,"y":6,"width":950,"height":387,"text":"**Get list of source tables**\n\nQuery cloud storage to get a list of paths containing CDC files","colour":"e6e63c"}},"variables":{"protocol":{"definition":{"name":"protocol","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"host_path":{"definition":{"name":"host_path","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":null},"table_list_sql":{"definition":{"name":"table_list_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"cloud_storage_url":{"definition":{"name":"cloud_storage_url","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"azure_key_vault_url":{"definition":{"name":"azure_key_vault_url","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"storage_account_key_secret":{"definition":{"name":"storage_account_key_secret","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""}},"grids":{"source_tables":{"definition":{"name":"source_tables","scope":"TASKBATCH","definitions":[{"name":"source_database","type":"TEXT"},{"name":"source_schema","type":"TEXT"},{"name":"source_table","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]}}}},"transformationJobs":{}},{"metadata":{"identifier":{"packageName":"Matillion.Data Loader.CDC","name":"Tag All CDC Files (Exchange)","revision":1,"type":"DYNAMIC"},"rootJobReference":{"name":"Tag All CDC Files - 1 - Iterate over tables","type":"ORCHESTRATION","parameterMetadata":[{"slot":2,"variableName":"cloud_storage_url","variableType":"SCALAR","displayName":"Cloud Storage URL","description":"The URL of the location where the CDC change event files are stored. This should have the format:\n\ns3:////\nor\nazure://.blob.core.windows.net///\nor\ngs:////","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":3,"variableName":"azure_key_vault_url","variableType":"SCALAR","displayName":"Azure Key Vault URL","description":"Required for Azure Blob Storage\nThe URL of the Azure Key Vault used to store the Azure Blob Storage connection string. This is also called the vault's \"DNS Name\".","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":false},{"slot":4,"variableName":"storage_account_key_secret","variableType":"SCALAR","displayName":"Azure Blob Storage Account Key Secret","description":"Required for Azure Blob Storage\nName of the Azure Key Vault secret which stores the Azure Blob Storage account key.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":false},{"slot":5,"variableName":"tag_tables","variableType":"GRID","displayName":"Tables to Process","description":"Optional.\n\nIf left blank, the job will tag the Avro and schema.avsc files for every table in the pipeline.\n\nIf you would like the job to only process files for a subset of tables in the pipeline, enter the names of the tables into this grid variable.","defaultValue":[{"values":{"1":""}}],"defaultValueType":null,"required":false}]},"orchestrationJobs":["Tag All CDC Files - 1 - Iterate over tables","Tag All CDC Files - 1-1 - Get source tables","Tag All CDC Files - 1-2 - Tag all files for table","Tag All CDC Files - 1-2-1 - Remove current schema file"],"transformationJobs":[],"imageMetadata":{"componentIcon":"iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAABhklEQVR4Xu2WMUvDQBiGO7h00UVwOchdEqPgL3AQuovSC7p1dBAcFAcnQQc3ncTBTVBJSv0HIjj6E3QXEWdB1Kr1vXqB9LNNUpOAw73wQi933/s9d1dIKhUjLdd1R23bvoK/4E4OvziOs0XzU4XCdR3wCbdzuJshhJigPRKFol1VDPptOjeMkHGrcjjn03QuUUUD4EpnPc8bZ4xV6Zq+Khog5nd4k677pRIAHuF7+AN+Sz2JogGi/4CG6KjroGt7ZAAMgAEwAGUB4K3YxPgSHqNre1QWQGb9GwD4AsfW+KuxgQeVg99Tyy1W9UOxRHv1FQoWNEARflJvP//cZTKwnmmvgdI7OEXAGXynwjC+0eOsPsbHyIzKGxogLgQdKQBArdC5rMoFgMaHCkDvajXJ2DGL6mQg9vyQnyjLkLcA0I7GP8+stXifgcLRSw2QasDORXX1QDT8gG8o10O+IwP+Go27Dq35eJ9EIXgRIPvwQZIBMUlrlXJdQREyALXr2ohs2h59bhTXNwiXYz3+XiDPAAAAAElFTkSuQmCC","smallIcon":"iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAA1klEQVR4XmNgoAZQUlLqA+K/BPAtbW1tNnS9YKCoqLgfiO3QxZEB0IDXysrKskAmE7oc0QYA8ScgvsGAbgixBqipqYkA6Y9SUlJcKJIjz4CLMjIynCiSpBiALg4GQMnNQHwQiNfiwZ8NE1VEA5fKB6HrB7lAHJhIAoGKbgHZJUA6CB2rqKgYei5R4QtaKl+Frh8OgIbsACo+pARxEQwXgOQClivWBC5V6AtcpnAgcLlCR/AyRT10/QwgPwJd4IaMgYbqwOQJuoAQSJivwBGwQtENXZwiAAC1wGPcQi+65QAAAABJRU5ErkJggg=="},"description":"When running Sync All Tables, it's possible to set the parameter Tag Processed Files to apply a tag on files in cloud storage, after the data has been loaded into the target tables. This makes it possible to create a lifecycle management policy to manage cloud storage costs.\n\nIf the pipeline has been running with this parameter disabled, and you would like to turn on tagging, then\n\n1. Allow Sync All Tables to complete execution, then pause the schedule\n2. Run this job (Tag All CDC Files) to tag all the existing files\n3. Update Sync All Tables with Tag Processed Files = Y\n3. Restart the schedule for Sync All Tables\n\nNote: For Azure Blob Storage, Blob Index Tags are not supported on Storage Accounts with a hierarchical namespace enabled.","helpHtml":"

Tag All CDC Files

When running Sync All Tables, it's possible to set the parameter Tag Processed Files to apply a tag on files in cloud storage, after the data has been loaded into the target tables. This makes it possible to create a lifecycle management policy to manage cloud storage costs.\n\nIf the pipeline has been running with this parameter disabled, and you would like to turn on tagging, then\n\n1. Allow Sync All Tables to complete execution, then pause the schedule\n2. Run this job (Tag All CDC Files) to tag all the existing files\n3. Update Sync All Tables with Tag Processed Files = Y\n3. Restart the schedule for Sync All Tables\n\nNote: For Azure Blob Storage, Blob Index Tags are not supported on Storage Accounts with a hierarchical namespace enabled.

Properties

PropertyTypeDescription
Cloud Storage URLVariableThe URL of the location where the CDC change event files are stored. This should have the format:\n\ns3:////\nor\nazure://.blob.core.windows.net///\nor\ngs:////
Azure Key Vault URLVariableRequired for Azure Blob Storage\nThe URL of the Azure Key Vault used to store the Azure Blob Storage connection string. This is also called the vault's \"DNS Name\".
Azure Blob Storage Account Key SecretVariableRequired for Azure Blob Storage\nName of the Azure Key Vault secret which stores the Azure Blob Storage account key.
Tables to ProcessGridOptional.\n\nIf left blank, the job will tag the Avro and schema.avsc files for every table in the pipeline.\n\nIf you would like the job to only process files for a subset of tables in the pipeline, enter the names of the tables into this grid variable.
","created":1709726627103},"orchestrationJobs":{"Tag All CDC Files - 1 - Iterate over tables":{"id":-2,"revision":3,"created":1709726528748,"timestamp":1709726528748,"components":{"49211":{"id":49211,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":-1520,"y":0,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[49227],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Start","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"49212":{"id":49212,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":-320,"y":0,"width":32,"height":32,"inputConnectorIDs":[49446],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Success - All files tagged"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"49213":{"id":49213,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":-880,"y":0,"width":32,"height":32,"inputConnectorIDs":[49796],"outputSuccessConnectorIDs":[49811],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"protocol","mapTo":"protocol","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"bucket","mapTo":"bucket","validationStatus":null,"validationMessage":null},"3":{"slot":3,"fromId":null,"fromName":"prefix","mapTo":"prefix","validationStatus":null,"validationMessage":null},"4":{"slot":4,"fromId":null,"fromName":"account","mapTo":"account","validationStatus":null,"validationMessage":null},"5":{"slot":5,"fromId":null,"fromName":"container","mapTo":"container","validationStatus":null,"validationMessage":null}},"gridExportMappings":{"1":{"slot":1,"fromGrid":"source_tables","toGrid":"source_tables","columnMapToFrom":{"source_table":"source_table","source_schema":"source_schema","source_database":"source_database"},"validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Get list of source tables in pipeline"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Tag All CDC Files - 1-1 - Get source tables"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"cloud_storage_url"},"2":{"slot":2,"type":"STRING","value":"${cloud_storage_url}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"azure_key_vault_url"},"2":{"slot":2,"type":"STRING","value":"${azure_key_vault_url}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"storage_account_key_secret"},"2":{"slot":2,"type":"STRING","value":"${storage_account_key_secret}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Tag All CDC Files (Exchange)"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Data Loader.CDC"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"49218":{"id":49218,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":-1360,"y":0,"width":32,"height":32,"inputConnectorIDs":[49227],"outputSuccessConnectorIDs":[49750],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Print version"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"print('Version: 1.0.2')"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"49383":{"id":49383,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"ITERATE","implementationID":-424773870,"x":-480,"y":0,"width":32,"height":16,"inputConnectorIDs":[49815],"outputSuccessConnectorIDs":[49446],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Iterate over table list"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Grid Variable","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"tag_tables"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Grid Variable Mapping","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"source_database"},"2":{"slot":2,"type":"STRING","value":"source_database"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"source_schema"},"2":{"slot":2,"type":"STRING","value":"source_schema"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"source_table"},"2":{"slot":2,"type":"STRING","value":"source_table"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Break on Failure","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Yes"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Concurrency","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sequential"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Stop on Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"No"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Stop Condition Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":false,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Condition","elements":{},"visible":false,"status":{},"columnNames":[]},"10":{"slot":10,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":false,"status":{},"columnNames":[]},"11":{"slot":11,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"999":{"slot":999,"name":"Record Values In Task History","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Yes"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[49424],"inputIterationConnectorIDs":[]},"49421":{"id":49421,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":-480,"y":32,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Tag all files for table"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Tag All CDC Files - 1-2 - Tag all files for table"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"protocol"},"2":{"slot":2,"type":"STRING","value":"${protocol}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"bucket"},"2":{"slot":2,"type":"STRING","value":"${bucket}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"prefix"},"2":{"slot":2,"type":"STRING","value":"${prefix}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"source_database"},"2":{"slot":2,"type":"STRING","value":"${source_database}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"source_schema"},"2":{"slot":2,"type":"STRING","value":"${source_schema}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"source_table"},"2":{"slot":2,"type":"STRING","value":"${source_table}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"azure_key_vault_url"},"2":{"slot":2,"type":"STRING","value":"${azure_key_vault_url}"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"storage_account_key_secret"},"2":{"slot":2,"type":"STRING","value":"${storage_account_key_secret}"}}},"9":{"slot":9,"values":{"1":{"slot":1,"type":"STRING","value":"account"},"2":{"slot":2,"type":"STRING","value":"${account}"}}},"10":{"slot":10,"values":{"1":{"slot":1,"type":"STRING","value":"container"},"2":{"slot":2,"type":"STRING","value":"${container}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Tag All CDC Files (Exchange)"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Data Loader.CDC"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[49424]},"49743":{"id":49743,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":-1120,"y":0,"width":32,"height":32,"inputConnectorIDs":[49750],"outputSuccessConnectorIDs":[49796],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Validate input"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"if not cloud_storage_url[-1:] == '/':\n cloud_storage_url = cloud_storage_url + '/'\n \nprint(f'cloud_storage_url: {cloud_storage_url!r}')\ncontext.updateVariable('cloud_storage_url', cloud_storage_url)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"49810":{"id":49810,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":-720,"y":0,"width":32,"height":32,"inputConnectorIDs":[49811],"outputSuccessConnectorIDs":[49815],"outputFailureConnectorIDs":[50008],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Select tables for tagging"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"from collections import namedtuple\nfrom pprint import pprint\n\n\nSourceTable = namedtuple('SourceTable', 'database schema table')\n\n\ndef convert_list_to_set(table_list):\n return set(SourceTable(d, s, t) for d, s, t in table_list)\n\n\nsource_tables = context.getGridVariable('source_tables') or []\ntag_tables = context.getGridVariable('tag_tables') or []\n\nif not source_tables:\n raise Exception('Unable to proceed. No tables found in cloud storage.')\n\nif not tag_tables:\n print(\"No tables supplied in parameter 'Tables to Process', so tagging files from all tables in pipeline.\")\n tag_tables = source_tables\n\nelse:\n source_table_set = convert_list_to_set(source_tables)\n tag_table_set = convert_list_to_set(tag_tables)\n \n missing_tables = tag_table_set - source_table_set\n \n if missing_tables:\n print(\"The following tables from the input grid variable 'Tables to Process' were not found in cloud storage:\")\n pprint(missing_tables)\n print()\n print('Tables found in cloud storage:')\n pprint(source_table_set)\n print()\n \n raise Exception(\n 'Unable to proceed. Some of the tables from the input grid '\n \"variable 'Tables to Process' could not be found in cloud storage.\"\n )\n \nprint('tag_tables:')\npprint(tag_tables)\ncontext.updateGridVariable('tag_tables', tag_tables)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"50005":{"id":50005,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":-724,"y":154,"width":32,"height":32,"inputConnectorIDs":[50008],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Failure - Issue identifying tables to process"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"49446":{"id":49446,"sourceID":49383,"targetID":49212},"49750":{"id":49750,"sourceID":49218,"targetID":49743},"49796":{"id":49796,"sourceID":49743,"targetID":49213},"49811":{"id":49811,"sourceID":49213,"targetID":49810},"49815":{"id":49815,"sourceID":49810,"targetID":49383}},"failureConnectors":{"50008":{"id":50008,"sourceID":49810,"targetID":50005}},"unconditionalConnectors":{"49227":{"id":49227,"sourceID":49211,"targetID":49218}},"trueConnectors":{},"falseConnectors":{},"iterationConnectors":{"49424":{"id":49424,"sourceID":49383,"targetID":49421}},"noteConnectors":{},"canUndo":true,"undoCommand":"Set Parameter","undoCreated":1709726522518,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"49202":{"id":49202,"x":-2235,"y":-473,"width":628,"height":228,"text":"**Build a list of all tables in CDC pipeline, so that existing files can be tagged**\n\nWhen running __Sync All Tables__, it's possible to set the parameter __Tag Processed Files__ to apply a tag on files in cloud storage, after the data has been loaded into the target tables. This makes it possible to create a lifecycle management policy to manage cloud storage costs.\n\nIf the pipeline has been running with this parameter disabled, and you would like to turn on tagging, then\n\n1. Allow Sync All Tables to complete execution, then pause the schedule\n2. Run this job (Tag All CDC Files) to tag all the existing files\n3. Update Sync All Tables with Tag Processed Files = Y\n3. Restart the schedule for Sync All Tables","colour":"00ce4f"},"49205":{"id":49205,"x":-2234,"y":-154,"width":630,"height":231,"text":"**Change log**\n\n1.0.0 -----------------------------------------------------------------------------------------------------------\n- Initial version\n\n1.0.1 -----------------------------------------------------------------------------------------------------------\n- Ensure current schema.avsc file is not tagged\n\n1.0.2 -----------------------------------------------------------------------------------------------------------\n- Increased Python Script timeout to 600 seconds for tagging files on Azure\n- Added check for a hierarchical namespace before attempting to tag files in Azure","colour":"00ce4f"},"49206":{"id":49206,"x":-955,"y":-155,"width":312,"height":232,"text":"**Select tables to use for tagging**\n\n- Query cloud storage to obtain a list of all source tables in this CDC pipeline\n- If user has supplied a list of tables, check they do exist in this pipeline, then use this list for tagging\n- If the user has not supplied a list of tables, use the full pipeline table list for tagging","colour":"e6e63c"},"49208":{"id":49208,"x":-1437,"y":-155,"width":152,"height":229,"text":"**Print version**\n\nPrint shared job version to the task history, to help support cases","colour":"e6e63c"},"49791":{"id":49791,"x":-1194,"y":-154,"width":147,"height":228,"text":"**Clean up input parameters**\n\nValidate input parameters","colour":"e6e63c"},"49819":{"id":49819,"x":-555,"y":-155,"width":151,"height":228,"text":"**Iterate over table list, tagging all files under each table**","colour":"e6e63c"}},"variables":{"bucket":{"definition":{"name":"bucket","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"prefix":{"definition":{"name":"prefix","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"account":{"definition":{"name":"account","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"protocol":{"definition":{"name":"protocol","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"container":{"definition":{"name":"container","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"source_table":{"definition":{"name":"source_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"source_schema":{"definition":{"name":"source_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"source_database":{"definition":{"name":"source_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"cloud_storage_url":{"definition":{"name":"cloud_storage_url","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"azure_key_vault_url":{"definition":{"name":"azure_key_vault_url","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"storage_account_key_secret":{"definition":{"name":"storage_account_key_secret","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""}},"grids":{"tag_tables":{"definition":{"name":"tag_tables","scope":"TASKBATCH","definitions":[{"name":"source_database","type":"TEXT"},{"name":"source_schema","type":"TEXT"},{"name":"source_table","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]},"source_tables":{"definition":{"name":"source_tables","scope":"TASKBATCH","definitions":[{"name":"source_database","type":"TEXT"},{"name":"source_schema","type":"TEXT"},{"name":"source_table","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[]}}},"Tag All CDC Files - 1-2 - Tag all files for table":{"id":-2,"revision":4,"created":1709726467949,"timestamp":1709726467949,"components":{"49416":{"id":49416,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":0,"y":0,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[49473],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Start","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"49464":{"id":49464,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":160,"y":0,"width":32,"height":32,"inputConnectorIDs":[49473],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[49494],"outputFalseConnectorIDs":[49481],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Protocol is s3?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"protocol"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"s3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"49465":{"id":49465,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":320,"y":80,"width":32,"height":32,"inputConnectorIDs":[49481],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[49572],"outputFalseConnectorIDs":[49575],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Protocol is gs?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"protocol"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"gs"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"protocol"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"gcs"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Or"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"49493":{"id":49493,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":480,"y":0,"width":32,"height":32,"inputConnectorIDs":[49494],"outputSuccessConnectorIDs":[49656],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Get list of files - S3"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"\"\"\"Query S3 to get a list of all files for source table.\"\"\"\nimport re\nfrom pprint import pprint\nfrom typing import List\nfrom typing import Optional\n\nfrom boto3 import client as boto3_client # type: ignore\n\n\nif __package__:\n from . import *\n\n\ndef get_all_files(\n bucket: str,\n prefix: str,\n continuation_token: Optional[str] = None,\n) -> List[List[str]]:\n \"\"\"Get a list of all Avro files under prefix.\n\n Args:\n bucket: S3 bucket\n prefix: String with format\n 'database=/schema=/table=/'\n continuation_token: ContinuationToken indicates Amazon S3 that the list is being continued on this\n bucket with a token. ContinuationToken is obfuscated and is not a real key.\n\n Returns:\n List[List[str]]: List of keys for new files\n \"\"\"\n client = boto3_client('s3')\n\n if continuation_token:\n response = client.list_objects_v2(\n Bucket=bucket,\n Prefix=prefix,\n ContinuationToken=continuation_token,\n )\n else:\n response = client.list_objects_v2(\n Bucket=bucket,\n Prefix=prefix,\n )\n\n all_files = []\n file_pattern = re.compile(r'(?:^version=\\d+/year=\\d{4}/month=\\d{2}/day=\\d{2}/hour=\\d{2}/[\\w-]+(?:\\.avro)$|^version=\\d+/schema.avsc$)')\n\n for content in response.get('Contents', list()):\n current_key: str = content.get('Key', '')\n key_stub = current_key[len(prefix) :]\n\n if file_pattern.match(key_stub):\n all_files.append([current_key])\n\n if response.get('IsTruncated'):\n continuation_token = response.get('NextContinuationToken')\n all_files.extend(\n get_all_files(bucket, prefix, continuation_token)\n )\n\n return all_files\n\n\ndef main() -> None:\n \"\"\"Query S3 to get a list of all files for source table.\"\"\"\n print(f'bucket: {bucket!r}')\n print(f'prefix: {prefix!r}')\n print(f'source_database: {source_database!r}')\n print(f'source_schema: {source_schema!r}')\n print(f'source_table: {source_table!r}')\n\n table_prefix = f'{prefix}database={source_database}/schema={source_schema}/table={source_table}/'\n print(f'table_prefix: {table_prefix!r}')\n print()\n\n all_files = get_all_files(bucket, table_prefix)\n if not all_files:\n print('No files found.')\n else:\n file_count = len(all_files)\n print(f'file_count: {file_count}')\n context.updateVariable('file_count', file_count)\n print()\n\n print('all_files:')\n pprint(all_files)\n context.updateGridVariable('all_files', all_files)\n\n\nif __name__ == \"__main__\":\n main()\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"49502":{"id":49502,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":816,"y":0,"width":32,"height":32,"inputConnectorIDs":[49661],"outputSuccessConnectorIDs":[49609],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Tag files - S3"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"\"\"\"Apply matillion-cdc-processed tag to list of files in S3.\"\"\"\nfrom boto3 import client as boto3_client\n\n\ndef get_tags(client, bucket, object_key):\n \"\"\"Call S3 API to retrieve current tags for object.\"\"\"\n try:\n response = client.get_object_tagging(\n Bucket=bucket,\n Key=object_key,\n )\n except client.exceptions.NoSuchKey:\n raise\n\n return response['TagSet']\n\n\ndef put_tags(client, bucket, object_key, tag_set):\n \"\"\"Call S3 API to set new tags against an object.\"\"\"\n client.put_object_tagging(\n Bucket=bucket,\n Key=object_key,\n Tagging={\n 'TagSet': tag_set\n },\n )\n\n\ndef available_tags(tag_set, tag_key, max_tags):\n \"\"\"Check whether the tag_set has enough room for an additional tag.\"\"\"\n if len(tag_set) < max_tags:\n return True\n\n # if we're at the max, check if our tag is already one of them\n for tag in tag_set:\n if tag['Key'] == tag_key:\n return True\n else:\n return False\n\n\ndef tag_already_exists(tag_set, tag_key, tag_value):\n \"\"\"Check if our tag key and value already exist in tag_set.\"\"\"\n for tag in tag_set:\n if tag['Key'] == tag_key and tag['Value'] == tag_value:\n return True\n return False\n\n\ndef update_tag_set(tag_set, tag_key, tag_value):\n \"\"\"Add or update our tag_key to tag_set.\"\"\"\n for tag in tag_set:\n if tag['Key'] == tag_key:\n tag['Value'] = tag_value\n break\n else:\n tag_set.append(\n {\n 'Key': tag_key,\n 'Value': tag_value\n }\n )\n\n\nclient = boto3_client('s3')\n\nprint(f'bucket: {bucket!r}')\n\ns3_max_tags = 10\ntag_key = 'matillion_cdc_processed'\ntag_value = 'true'\n\nall_files = context.getGridVariable('all_files') or []\n\nfor [object_key] in all_files:\n\n print()\n print(f'Reading existing tags for {object_key!r}')\n try:\n tag_set = get_tags(client, bucket, object_key)\n except client.exceptions.NoSuchKey:\n print('Key does not exist')\n continue\n\n if not available_tags(tag_set, tag_key, s3_max_tags):\n print(f\"Skipping, the maximum number of tags ({s3_max_tags}) already exist for {object_key!r}\")\n continue\n\n if tag_already_exists(tag_set, tag_key, tag_value):\n print(f\"Skipping, {tag_key!r} tag already exists for {object_key!r}\")\n continue\n\n update_tag_set(tag_set, tag_key, tag_value)\n\n print(f\"Adding {tag_key!r} tag to {object_key!r}\")\n put_tags(client, bucket, object_key, tag_set)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"49516":{"id":49516,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":816,"y":160,"width":32,"height":32,"inputConnectorIDs":[49745],"outputSuccessConnectorIDs":[49615],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Tag files - Azure"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"\"\"\"Apply matillion-cdc-processed tag to list of files in Azure Blob Storage.\"\"\"\ntry:\n from azure.core.exceptions import ResourceNotFoundError\n from azure.identity import ManagedIdentityCredential\n from azure.keyvault.secrets import SecretClient\n from azure.storage.blob import BlobServiceClient\nexcept ModuleNotFoundError as e:\n err_msg = (\n 'Module not found. The following Python3 packages are '\n 'required: [azure-identity, azure-keyvault-secrets, azure-storage-blob]'\n )\n raise ModuleNotFoundError(err_msg) from e\n\n\ndef get_storage_account_key(\n azure_key_vault_url: str, storage_account_key_secret: str\n) -> str:\n \"\"\"Get Storage Account key from Key Vault.\n\n Args:\n azure_key_vault_url: URL for the Azure Key Vault\n storage_account_key_secret: Name of the secret storing the Storage Account Key\n\n Returns:\n str: Storage Account key\n \"\"\"\n credential = ManagedIdentityCredential()\n key_vault_client = SecretClient(\n vault_url=azure_key_vault_url, credential=credential\n )\n key_vault_secret = key_vault_client.get_secret(storage_account_key_secret)\n\n return str(key_vault_secret.value)\n\n\ndef available_tags(tags, tag_key, max_tags):\n \"\"\"Check whether the tags dict has enough room for an additional tag.\"\"\"\n if len(tags) < max_tags:\n return True\n\n # if we're at the max, check if our tag is already one of them\n if tag_key in tags.keys():\n return True\n else:\n return False\n\n\ndef tag_already_exists(tags, tag_key, tag_value):\n \"\"\"Check if our tag key and value already exist in tags.\"\"\"\n if tag_key in tags.keys() and tags[tag_key] == tag_value:\n return True\n else:\n return False\n\n\nazure_max_tags = 10\ntag_key = 'matillion_cdc_processed'\ntag_value = 'true'\n\nprint(f'account: {account!r}')\nprint(f'container: {container!r}')\nprint()\n\nprint(\n f'Attempting to read the storage account key from the secret {storage_account_key_secret!r} '\n f'in the Key Vault {azure_key_vault_url!r}'\n)\naccount_key = get_storage_account_key(\n azure_key_vault_url, storage_account_key_secret\n)\nprint('Successfully read storage account key')\n\nblob_service_client = BlobServiceClient(\n account_url=f'https://{account}', credential=account_key\n)\n\n# Blob tags not supported on storage accounts with a hierarchical namespace\naccount_info = blob_service_client.get_account_information()\nif account_info.get('is_hns_enabled', False):\n err_msg = (\n f'The storage account {account!r} has a hierarchical namespace enabled. \\n'\n 'Blob Storage does not support Blob Index Tags on accounts with a hierarchical namespace enabled. \\n'\n 'For more details, see https://learn.microsoft.com/en-us/azure/storage/blobs/storage-feature-support-in-storage-accounts#standard-general-purpose-v2-accounts \\n'\n 'Please set the shared job parameter \"Tag Processed Files\" to \"N\" and re-run the job.'\n )\n raise Exception(err_msg)\n\nall_files = context.getGridVariable('all_files') or []\n\nfor [blob_name] in all_files:\n\n blob_client = blob_service_client.get_blob_client(container, blob_name)\n\n print()\n print(f'Reading existing tags for {blob_name!r}')\n try:\n tags = blob_client.get_blob_tags()\n except ResourceNotFoundError:\n print('Blob does not exist')\n continue\n\n if not available_tags(tags, tag_key, azure_max_tags):\n print(f\"Skipping, the maximum number of tags ({azure_max_tags}) already exist for {blob_name!r}\")\n continue\n\n if tag_already_exists(tags, tag_key, tag_value):\n print(f\"Skipping, {tag_key!r} tag already exists for {blob_name!r}\")\n continue\n\n tags[tag_key] = tag_value\n\n print(f\"Adding {tag_key!r} tag to {blob_name!r}\")\n blob_client.set_blob_tags(tags)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"600"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"49517":{"id":49517,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":816,"y":80,"width":32,"height":32,"inputConnectorIDs":[49739],"outputSuccessConnectorIDs":[49612],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Tag files - GCS"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"\"\"\"Set Custom-Time metadata on list of files in GCS.\"\"\"\nfrom datetime import datetime, timezone\nfrom google.cloud import storage\n\n\nprint(f'bucket: {bucket!r}')\n\ncustom_time = datetime.now(timezone.utc)\n\nall_files = context.getGridVariable('all_files') or []\n\nstorage_client = storage.Client()\nbucket_object = storage_client.bucket(bucket)\n\nfor [blob_name] in all_files:\n\n print()\n print(f\"Reading 'Custom-Time' metadata for {blob_name!r}\")\n blob = bucket_object.get_blob(blob_name)\n if not blob:\n print('Blob does not exist')\n continue\n\n current_custom_time = blob.custom_time\n\n if current_custom_time and current_custom_time > custom_time:\n print(f\"Skipping, existing 'Custom-Time' value ({current_custom_time}) > new value ({custom_time})\")\n continue\n\n if not current_custom_time:\n print(f\"Setting 'Custom-Time' to {custom_time}\")\n elif blob_name[-11:] == 'schema.avsc':\n print(\"Skipping, 'schema.avsc' already has 'Custom-Time' set\")\n else:\n print(f\"Updating 'Custom-Time' from {current_custom_time} to {custom_time}\")\n\n blob.custom_time = custom_time\n blob.patch()\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"49567":{"id":49567,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":477,"y":80,"width":32,"height":32,"inputConnectorIDs":[49572],"outputSuccessConnectorIDs":[49736],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Get list of files - GCS"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"\"\"\"Query GCS to get a list of all files for source table.\"\"\"\nimport re\nfrom pprint import pprint\nfrom typing import List\n\nfrom google.cloud import storage # type: ignore\n\n\nif __package__:\n from . import *\n\n\ndef get_all_files(bucket: str, prefix: str) -> List[List[str]]:\n \"\"\"Get a list of all Avro files under prefix.\n\n Args:\n bucket: Google Cloud Storage bucket\n prefix: String with format\n 'database=/schema=/table=
/version=/'\n\n Returns:\n List[List[str]]: List of keys for new files\n \"\"\"\n gcs_client = storage.Client()\n\n blob_iterator = gcs_client.list_blobs(\n bucket_or_name=bucket, prefix=prefix\n )\n\n all_files = []\n file_pattern = re.compile(r'(?:^version=\\d+/year=\\d{4}/month=\\d{2}/day=\\d{2}/hour=\\d{2}/[\\w-]+(?:\\.avro)$|^version=\\d+/schema.avsc$)')\n\n for blob in blob_iterator:\n current_key: str = blob.name\n key_stub = current_key[len(prefix) :]\n\n if file_pattern.match(key_stub):\n all_files.append([current_key])\n\n return all_files\n\n\ndef main() -> None:\n \"\"\"Query GCS to get a list of all files for source table.\"\"\"\n print(f'bucket: {bucket!r}')\n print(f'prefix: {prefix!r}')\n print(f'source_database: {source_database!r}')\n print(f'source_schema: {source_schema!r}')\n print(f'source_table: {source_table!r}')\n\n table_prefix = f'{prefix}database={source_database}/schema={source_schema}/table={source_table}/'\n print(f'table_prefix: {table_prefix!r}')\n print()\n\n all_files = get_all_files(bucket, table_prefix)\n if not all_files:\n print('No files found.')\n else:\n file_count = len(all_files)\n print(f'file_count: {file_count}')\n context.updateVariable('file_count', file_count)\n print()\n\n print('all_files:')\n pprint(all_files)\n context.updateGridVariable('all_files', all_files)\n\n\nif __name__ == \"__main__\":\n main()\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"49569":{"id":49569,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":480,"y":157,"width":32,"height":32,"inputConnectorIDs":[49575],"outputSuccessConnectorIDs":[49742],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Get list of files - Azure"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"\"\"\"Query Azure Blob Storage to get a list of all files for source table.\"\"\"\ntry:\n from azure.identity import ManagedIdentityCredential\n from azure.keyvault.secrets import SecretClient\n from azure.storage.blob import BlobServiceClient\nexcept ModuleNotFoundError as e:\n err_msg = (\n 'Module not found. The following Python3 packages are '\n 'required: [azure-identity, azure-keyvault-secrets, azure-storage-blob]'\n )\n raise ModuleNotFoundError(err_msg) from e\n\nfrom pprint import pprint\nimport re\nfrom typing import List\n\n\nif __package__:\n from . import *\n\n\ndef get_storage_account_key(\n azure_key_vault_url: str, storage_account_key_secret: str\n) -> str:\n \"\"\"Get Storage Account key from Key Vault.\n\n Args:\n azure_key_vault_url: URL for the Azure Key Vault\n storage_account_key_secret: Name of the secret storing the Storage Account Key\n\n Returns:\n str: Storage Account key\n \"\"\"\n credential = ManagedIdentityCredential()\n key_vault_client = SecretClient(\n vault_url=azure_key_vault_url, credential=credential\n )\n key_vault_secret = key_vault_client.get_secret(storage_account_key_secret)\n\n return str(key_vault_secret.value)\n\n\ndef get_all_files(\n account_key: str, account: str, container: str, prefix: str\n) -> List[List[str]]:\n \"\"\"Get a list of all Avro files under prefix.\n\n Args:\n account_key: Key for Azure Storage Account\n account: Azure Storage Account\n container: Container in Storage account\n prefix: String with format\n 'database=/schema=/table=
/'\n\n Returns:\n List[List[str]]: List of keys for new files\n \"\"\"\n # get list of blobs in container which start with prefix\n blob_service_client = BlobServiceClient(\n account_url=f'https://{account}', credential=account_key\n )\n container_client = blob_service_client.get_container_client(container)\n blobs = container_client.list_blobs(name_starts_with=prefix)\n\n all_files = []\n file_pattern = re.compile(r'(?:^version=\\d+/year=\\d{4}/month=\\d{2}/day=\\d{2}/hour=\\d{2}/[\\w-]+(?:\\.avro)$|^version=\\d+/schema.avsc$)')\n\n for blob in blobs:\n current_key: str = blob.name\n key_stub = current_key[len(prefix) :]\n\n if file_pattern.match(key_stub):\n all_files.append([current_key])\n\n return all_files\n\n\ndef main() -> None:\n \"\"\"Query Azure Blob Storage to get a list of all files for source table.\"\"\"\n print(f'account: {account!r}')\n print(f'container: {container!r}')\n print(f'prefix: {prefix!r}')\n print(f'source_database: {source_database!r}')\n print(f'source_schema: {source_schema!r}')\n print(f'source_table: {source_table!r}')\n\n table_prefix = f'{prefix}database={source_database}/schema={source_schema}/table={source_table}/'\n print(f'table_prefix: {table_prefix!r}')\n print()\n\n # lookup storage account key from Azure Key Vault\n print(\n f'Attempting to read the storage account key from the secret {storage_account_key_secret!r} '\n f'in the Key Vault {azure_key_vault_url!r}'\n )\n account_key = get_storage_account_key(\n azure_key_vault_url, storage_account_key_secret\n )\n print('Successfully read storage account key')\n print()\n\n all_files = get_all_files(account_key, account, container, table_prefix)\n if not all_files:\n print('No files found.')\n else:\n file_count = len(all_files)\n print(f'file_count: {file_count}')\n context.updateVariable('file_count', file_count)\n print()\n\n print('all_files:')\n pprint(all_files)\n context.updateGridVariable('all_files', all_files)\n\n\nif __name__ == \"__main__\":\n main()\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"600"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"49603":{"id":49603,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":1136,"y":0,"width":32,"height":32,"inputConnectorIDs":[49618],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Success"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"49604":{"id":49604,"inputCardinality":"MANY","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1343684451,"x":976,"y":0,"width":32,"height":32,"inputConnectorIDs":[49609,49612,49615],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[49618],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Join paths"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"49655":{"id":49655,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":640,"y":0,"width":32,"height":32,"inputConnectorIDs":[49656],"outputSuccessConnectorIDs":[49661],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"gridExportMappings":{"1":{"slot":1,"fromGrid":"all_files","toGrid":"all_files","columnMapToFrom":{"filepath":"filepath"},"validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Remove current schema file - S3"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Tag All CDC Files - 1-2-1 - Remove current schema file"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"all_files"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"filepath"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"all_files"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Tag All CDC Files (Exchange)"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Data Loader.CDC"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"49717":{"id":49717,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":640,"y":160,"width":32,"height":32,"inputConnectorIDs":[49742],"outputSuccessConnectorIDs":[49745],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"gridExportMappings":{"1":{"slot":1,"fromGrid":"all_files","toGrid":"all_files","columnMapToFrom":{"filepath":"filepath"},"validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Remove current schema file - Azure"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Tag All CDC Files - 1-2-1 - Remove current schema file"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"all_files"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"filepath"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"all_files"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Tag All CDC Files (Exchange)"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Data Loader.CDC"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"49720":{"id":49720,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":640,"y":80,"width":32,"height":32,"inputConnectorIDs":[49736],"outputSuccessConnectorIDs":[49739],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"gridExportMappings":{"1":{"slot":1,"fromGrid":"all_files","toGrid":"all_files","columnMapToFrom":{"filepath":"filepath"},"validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Remove current schema file - GCS"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Tag All CDC Files - 1-2-1 - Remove current schema file"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"all_files"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"filepath"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"all_files"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Tag All CDC Files (Exchange)"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Data Loader.CDC"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"49609":{"id":49609,"sourceID":49502,"targetID":49604},"49612":{"id":49612,"sourceID":49517,"targetID":49604},"49615":{"id":49615,"sourceID":49516,"targetID":49604},"49656":{"id":49656,"sourceID":49493,"targetID":49655},"49661":{"id":49661,"sourceID":49655,"targetID":49502},"49736":{"id":49736,"sourceID":49567,"targetID":49720},"49739":{"id":49739,"sourceID":49720,"targetID":49517},"49742":{"id":49742,"sourceID":49569,"targetID":49717},"49745":{"id":49745,"sourceID":49717,"targetID":49516}},"failureConnectors":{},"unconditionalConnectors":{"49473":{"id":49473,"sourceID":49416,"targetID":49464},"49618":{"id":49618,"sourceID":49604,"targetID":49603}},"trueConnectors":{"49494":{"id":49494,"sourceID":49464,"targetID":49493},"49572":{"id":49572,"sourceID":49465,"targetID":49567}},"falseConnectors":{"49481":{"id":49481,"sourceID":49464,"targetID":49465},"49575":{"id":49575,"sourceID":49465,"targetID":49569}},"iterationConnectors":{},"noteConnectors":{},"canUndo":true,"undoCommand":"Set Parameter","undoCreated":1709726456125,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"49633":{"id":49633,"x":84,"y":-155,"width":789,"height":391,"text":"**Get list of all files for table, then apply tags**\n\nGet list of all Avro and schema.avsc files for the current source table.","colour":"e6e63c"}},"variables":{"bucket":{"definition":{"name":"bucket","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"prefix":{"definition":{"name":"prefix","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"account":{"definition":{"name":"account","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"protocol":{"definition":{"name":"protocol","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"container":{"definition":{"name":"container","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"source_table":{"definition":{"name":"source_table","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"source_schema":{"definition":{"name":"source_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"source_database":{"definition":{"name":"source_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"azure_key_vault_url":{"definition":{"name":"azure_key_vault_url","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"storage_account_key_secret":{"definition":{"name":"storage_account_key_secret","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""}},"grids":{"all_files":{"definition":{"name":"all_files","scope":"TASKBATCH","definitions":[{"name":"filepath","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[{"values":["test/tmp/version-sf/ec02bc0be1764522ab84ea3e88ad7477/database=postgres/schema=sol_eng/table=item_tx_1/version=0/schema.avsc"]},{"values":["test/tmp/version-sf/ec02bc0be1764522ab84ea3e88ad7477/database=postgres/schema=sol_eng/table=item_tx_1/version=1/schema.avsc"]},{"values":["test/tmp/version-sf/ec02bc0be1764522ab84ea3e88ad7477/database=postgres/schema=sol_eng/table=item_tx_1/version=1/year=2021/month=09/day=30/hour=13/0000000001.avro"]},{"values":["test/tmp/version-sf/ec02bc0be1764522ab84ea3e88ad7477/database=postgres/schema=sol_eng/table=item_tx_1/version=1/year=2021/month=09/day=30/hour=13/0000000002.avro"]}]}}},"Tag All CDC Files - 1-2-1 - Remove current schema file":{"id":-2,"revision":1,"created":1709542096647,"timestamp":1709542096647,"components":{"49615":{"id":49615,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":0,"y":0,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[49643],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Start","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"49630":{"id":49630,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":160,"y":0,"width":32,"height":32,"inputConnectorIDs":[49643],"outputSuccessConnectorIDs":[49646],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Remove current schema.avsc file"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"import re\n\n\nall_files = context.getGridVariable('all_files') or []\n\nversion_pattern = re.compile(r'^.+/version=(\\d+)/schema.avsc$')\nschema_files = {int(version_pattern.match(f)[1]): f for [f] in all_files if f.endswith('/schema.avsc')}\n\nif not schema_files:\n print('No schema.avsc files found')\n exit()\n\ncurrent_schema_file = schema_files[max(schema_files.keys())]\nprint(f'current_schema_file: {current_schema_file}')\nprint()\n\nall_files = [[f] for [f] in all_files if f != current_schema_file]\nprint('all_files:')\nfor row in all_files:\n print(\"\\t\".join(str(col) for col in row))\n context.updateGridVariable('all_files', all_files)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"49645":{"id":49645,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":320,"y":0,"width":32,"height":32,"inputConnectorIDs":[49646],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Success"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"49646":{"id":49646,"sourceID":49630,"targetID":49645}},"failureConnectors":{},"unconditionalConnectors":{"49643":{"id":49643,"sourceID":49615,"targetID":49630}},"trueConnectors":{},"falseConnectors":{},"iterationConnectors":{},"noteConnectors":{},"canUndo":false,"undoCommand":"","undoCreated":0,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"49667":{"id":49667,"x":86,"y":-155,"width":149,"height":229,"text":"**Ensure current schema.avsc file is not in the tagging list**\n\nSync Single Table requires the current schema.avsc file, so ensure it isn't tagged as processed.","colour":"e6e63c"}},"variables":{},"grids":{"all_files":{"definition":{"name":"all_files","scope":"TASKBATCH","definitions":[{"name":"filepath","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]}}},"Tag All CDC Files - 1-1 - Get source tables":{"id":-2,"revision":2,"created":1709726480091,"timestamp":1709726480091,"components":{"49314":{"id":49314,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":720,"y":240,"width":32,"height":32,"inputConnectorIDs":[49331],"outputSuccessConnectorIDs":[49330],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Get list of tables from GCS"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"\"\"\"Get a list of source tables from Google Cloud Storage.\"\"\"\nfrom pprint import pprint\nfrom typing import List\nfrom typing import Set\n\nfrom google.cloud import storage # type: ignore\n\nfrom . import *\n\n\ndef get_source_table_paths(bucket: str, prefix: str) -> List[str]:\n \"\"\"Iterate over database and schema folders to get a list of table paths.\n\n Args:\n bucket: Google Cloud Storage bucket\n prefix: Prefix for object paths in bucket\n\n Returns:\n List[str]: List of strings of format 'database=/schema=/table=
/'\n \"\"\"\n client = storage.Client()\n\n source_table_paths: List[str] = []\n database_paths = get_child_folders(client, bucket, prefix)\n\n for database in database_paths:\n schema_paths = get_child_folders(client, bucket, database)\n\n for schema in schema_paths:\n source_table_paths.extend(get_child_folders(client, bucket, schema))\n\n return source_table_paths\n\n\ndef get_child_folders(\n client: storage.client.Client, bucket: str, prefix: str\n) -> Set[str]:\n \"\"\"Get list of child folders under '/'.\n\n Args:\n client: Google storage client\n bucket: Google Cloud Storage bucket\n prefix: Prefix for object paths in bucket\n\n Returns:\n Set[str]: List of strings of format '/'\n \"\"\"\n blob_iterator = client.list_blobs(\n bucket_or_name=bucket,\n prefix=prefix,\n delimiter='/',\n include_trailing_delimiter=True,\n )\n\n # exhaust generator, without storing values\n # blob_iterator will build a set of prefixes as it yields items\n for _ in blob_iterator:\n pass\n\n return blob_iterator.prefixes # type: ignore\n\n\ndef extract_source_tables(paths: List[str], prefix: str) -> List[List[str]]:\n \"\"\"Convert list of table paths into lists of [, ,
].\n\n Args:\n paths: List of strings of format 'database=/schema=/table=
/'\n prefix: Prefix for object paths in bucket\n\n Returns:\n List[List[str]]: List of lists of format [, ,
]\n \"\"\"\n source_tables = []\n for path in paths:\n try:\n source_tables.append(split_table_path(path, prefix))\n except IndexError:\n # ignore paths which do not contain '='\n pass\n\n return source_tables\n\n\ndef split_table_path(path: str, prefix: str) -> List[str]:\n \"\"\"Split table path into list of [, ,
].\n\n Args:\n path: String of format 'database=/schema=/table=
/'\n prefix: Prefix for object paths in bucket\n\n Returns:\n List[str]: List of strings of format [, ,
]\n\n Raises:\n IndexError: If any folder does not contain '='\n \"\"\"\n key_value_list = path[len(prefix) : -1].split('/')\n\n value_list = []\n for key_value in key_value_list:\n try:\n value_list.append(key_value.split('=', 1)[1])\n except IndexError:\n raise\n\n return value_list\n\n\ndef split_host_path(host_path: str) -> List[str]:\n \"\"\"Split host_path into host and path.\n\n Args:\n host_path: String of format '//'\n\n Returns:\n List[str]: List of format ['', '/']\n \"\"\"\n return host_path.split('/', 1)\n\n\ndef main() -> None:\n \"\"\"Get a list of source tables from Google Cloud Storage.\"\"\"\n bucket, prefix = split_host_path(host_path)\n \n print(f'bucket: {bucket!r}')\n context.updateVariable('bucket', bucket)\n\n print(f'prefix: {prefix!r}')\n context.updateVariable('prefix', prefix)\n print()\n\n # get a list of paths of form 'database=/schema=/table=
/'\n source_table_paths = get_source_table_paths(bucket, prefix)\n\n # convert to list of lists of form '[, ,
]'\n source_tables = extract_source_tables(source_table_paths, prefix)\n\n print('source_tables:')\n pprint(source_tables)\n context.updateGridVariable('source_tables', source_tables)\n\n\nif __name__ == \"__main__\":\n main()\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"49315":{"id":49315,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":560,"y":320,"width":32,"height":32,"inputConnectorIDs":[49333],"outputSuccessConnectorIDs":[49326],"outputFailureConnectorIDs":[49338],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Validate Azure parameters"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"print(f'azure_key_vault_url: {azure_key_vault_url!r}')\nprint(f'storage_account_key_secret: {storage_account_key_secret!r}')\n\nmissing_parameters = []\n\nif not azure_key_vault_url:\n missing_parameters.append('Azure Key Vault URL')\n \nif not storage_account_key_secret:\n missing_parameters.append('Azure Blob Storage Account Key Secret')\n \nif missing_parameters:\n err_msg = ('The following shared job parameters were not set, but are '\n f'required for Azure Blob Storage: {missing_parameters}')\n raise Exception(err_msg)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"49316":{"id":49316,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":560,"y":480,"width":32,"height":32,"inputConnectorIDs":[49338],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Failure - Azure Blob Storage parameters not set"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"49317":{"id":49317,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":74,"y":155,"width":32,"height":32,"inputConnectorIDs":[49336],"outputSuccessConnectorIDs":[49327],"outputFailureConnectorIDs":[49337],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Identify storage protocol"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"if not cloud_storage_url[-1:] == '/':\n cloud_storage_url = cloud_storage_url + '/'\n \nprint(f'cloud_storage_url: {cloud_storage_url!r}')\nprint()\n\n# split the cloud_storage_url into protocol and host_path\nurl_list = cloud_storage_url.split('://', 1)\n\n# check we could split the string and extract a protocol\nif len(url_list) == 1:\n err_msg = f'Unable to determine the protocol from the cloud_storage_url {cloud_storage_url!r}'\n raise Exception(err_msg)\n\n# unpack the list\nprotocol, host_path = url_list\nprotocol = protocol.lower()\n\n# check protocol is supported\nsupported_protocols = ('azure', 's3', 'gs', 'gcs')\nif protocol not in supported_protocols:\n err_msg = f'The protocol {protocol!r} was identifed from the cloud_storage_url {cloud_storage_url!r}, but the only supported protocols are {\", \".join(supported_protocols)}'\n raise Exception(err_msg)\n\nprint(f'protocol: {protocol!r}')\ncontext.updateVariable('protocol', protocol)\n\nprint(f'host_path: {host_path!r}')\ncontext.updateVariable('host_path', host_path)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"49318":{"id":49318,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":80,"y":480,"width":32,"height":32,"inputConnectorIDs":[49337],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Failure - Storage protocol not supported"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"49319":{"id":49319,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":234,"y":155,"width":32,"height":32,"inputConnectorIDs":[49327],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[49332],"outputFalseConnectorIDs":[49334],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Protocol is s3?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"protocol"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"s3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"49320":{"id":49320,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":720,"y":160,"width":32,"height":32,"inputConnectorIDs":[49332],"outputSuccessConnectorIDs":[49329],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Get list of tables from S3"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"import boto3\nfrom pprint import pprint\n\n\ndef get_common_prefixes(client, bucket, prefix, continuation_token=None):\n \"\"\"\n Wrapper for list_objects_v2 which returns all items when more than 1000\n \n Returns a list of dicts like [{'Prefix': 'niagara-shared-job-test/stage/postgres/'}]\n \"\"\"\n if continuation_token:\n response = client.list_objects_v2(\n Bucket=bucket, \n Prefix=prefix, \n Delimiter='/',\n ContinuationToken=continuation_token,\n )\n else:\n response = client.list_objects_v2(\n Bucket=bucket, \n Prefix=prefix, \n Delimiter='/',\n )\n \n common_prefixes = response.get('CommonPrefixes', list())\n \n if response.get('IsTruncated'):\n continuation_token = response.get('NextContinuationToken')\n common_prefixes.extend(\n get_common_prefixes(client, bucket, prefix, continuation_token)\n )\n \n return common_prefixes\n\n\ndef get_sub_paths(client, bucket, prefix):\n \"\"\"\n Extract list of paths from common_prefixes list\n \"\"\"\n common_prefixes = get_common_prefixes(client, bucket, prefix)\n sub_paths = [c.get('Prefix') for c in common_prefixes]\n return sub_paths\n\n\ndef split_bucket_prefix(s):\n \"\"\"\n Take base_path and return list of [bucket, path]\n \"\"\"\n path_list = s.split('/', 1)\n if len(path_list) == 1:\n return path_list.append('')\n else:\n return path_list\n\n\ndef split_path(s, prefix):\n \"\"\"\n Split path into list of [, ,
]\n \n Input path has format 'database=/schema=/table=
/'\n \"\"\"\n key_value_list = s[len(prefix):-1].split('/')\n return [key_value.split('=', 1)[1] for key_value in key_value_list]\n\n\nclient = boto3.client('s3')\n\n# split stage_base into bucket and prefix\nbucket, prefix = split_bucket_prefix(host_path)\n\nprint(f'bucket: {bucket!r}')\ncontext.updateVariable('bucket', bucket)\n\nprint(f'prefix: {prefix!r}')\ncontext.updateVariable('prefix', prefix)\n\n# get a list of paths of form '///
'\nsource_table_paths = []\ndatabase_paths = get_sub_paths(client, bucket, prefix)\nfor database in database_paths:\n schema_paths = get_sub_paths(client, bucket, database)\n for schema in schema_paths:\n source_table_paths.extend(get_sub_paths(client, bucket, schema))\n\n# convert to list of lists before storing as grid var, of form '[, ,
]'\nsource_tables = [\n split_path(path, prefix) for path in source_table_paths\n]\n\nprint('source_tables:')\npprint(source_tables)\ncontext.updateGridVariable('source_tables', source_tables)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"49321":{"id":49321,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":720,"y":320,"width":32,"height":32,"inputConnectorIDs":[49326],"outputSuccessConnectorIDs":[49328],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Get list of tables from Azure"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"try:\n from azure.identity import ManagedIdentityCredential\n from azure.keyvault.secrets import SecretClient\n from azure.storage.blob import BlobServiceClient\nexcept ModuleNotFoundError as e:\n err_msg = ('Module not found. The following Python3 packages are '\n 'required: [azure-identity, azure-keyvault-secrets, azure-storage-blob]'\n f'\\nException: {e!r}')\n raise ModuleNotFoundError(err_msg)\n \nfrom collections import namedtuple\nfrom pprint import pprint \n\n\n# create a named tuple so that we can use sets to get distinct list of tables\nTableReference = namedtuple('TableReference', 'database schema table')\n\n\ndef get_distinct_tables(blobs, prefix):\n \"\"\"\n Extract a list of distinct ['database', 'schema', 'table'] from a list of blobs\n \"\"\"\n all_table_refs = []\n for blob in blobs:\n \n # get blob name without 'prefix'\n blob_name = blob.name[len(prefix):]\n \n # try to split blob_name into ['database=', 'schema=', 'table=
', '....']\n blob_path_list = blob_name.split('/', 3)\n if len(blob_path_list) == 4:\n table_list = [key_value.split('=', 1)[1] for key_value in blob_path_list[:-1]]\n all_table_refs.append(\n TableReference(*table_list)\n )\n \n distinct_table_refs = set(all_table_refs)\n \n # convert named tuples to lists before returning\n return [list(t) for t in distinct_table_refs]\n \n\n# split host_path into account, container and prefix\naccount, container, prefix = host_path.split('/', 2)\n\nprint(f'account: {account!r}')\ncontext.updateVariable('account', account)\n\nprint(f'container: {container!r}')\ncontext.updateVariable('container', container)\n\nprint(f'prefix: {prefix!r}')\ncontext.updateVariable('prefix', prefix)\nprint()\n\n# lookup storage account key from Azure Key Vault\nprint(f'Attempting to read the storage account key from the secret {storage_account_key_secret!r} in the Key Vault {azure_key_vault_url!r}')\ncredential = ManagedIdentityCredential()\nkey_vault_client = SecretClient(vault_url=azure_key_vault_url, credential=credential)\nkey_vault_secret = key_vault_client.get_secret(storage_account_key_secret)\n\naccount_key = key_vault_secret.value\nprint('Successfully read storage account key')\nprint()\n\n# get list of blobs in container which start with prefix\nblob_service_client = BlobServiceClient(\n account_url=f'https://{account}',\n credential=account_key\n)\ncontainer_client = blob_service_client.get_container_client(container)\nblobs = container_client.list_blobs(name_starts_with=prefix)\n\n# get distinct list of source tables from blob names\nsource_tables = get_distinct_tables(blobs, prefix)\n\nprint('source_tables:')\npprint(source_tables)\ncontext.updateGridVariable('source_tables', source_tables)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"600"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"49322":{"id":49322,"inputCardinality":"MANY","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1343684451,"x":880,"y":160,"width":32,"height":32,"inputConnectorIDs":[49328,49329,49330],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[49335],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Join paths"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"49323":{"id":49323,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":-80,"y":160,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[49336],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"49324":{"id":49324,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":1040,"y":160,"width":32,"height":32,"inputConnectorIDs":[49335],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Success"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"49325":{"id":49325,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":384,"y":240,"width":32,"height":32,"inputConnectorIDs":[49334],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[49331],"outputFalseConnectorIDs":[49333],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Protocol is gs?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"protocol"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"gs"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"protocol"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"gcs"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Or"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"49326":{"id":49326,"sourceID":49315,"targetID":49321},"49327":{"id":49327,"sourceID":49317,"targetID":49319},"49328":{"id":49328,"sourceID":49321,"targetID":49322},"49329":{"id":49329,"sourceID":49320,"targetID":49322},"49330":{"id":49330,"sourceID":49314,"targetID":49322}},"failureConnectors":{"49337":{"id":49337,"sourceID":49317,"targetID":49318},"49338":{"id":49338,"sourceID":49315,"targetID":49316}},"unconditionalConnectors":{"49335":{"id":49335,"sourceID":49322,"targetID":49324},"49336":{"id":49336,"sourceID":49323,"targetID":49317}},"trueConnectors":{"49331":{"id":49331,"sourceID":49325,"targetID":49314},"49332":{"id":49332,"sourceID":49319,"targetID":49320}},"falseConnectors":{"49333":{"id":49333,"sourceID":49325,"targetID":49315},"49334":{"id":49334,"sourceID":49319,"targetID":49325}},"iterationConnectors":{},"noteConnectors":{},"canUndo":true,"undoCommand":"Set Parameter","undoCreated":1709542096647,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"49313":{"id":49313,"x":6,"y":6,"width":950,"height":387,"text":"**Get list of source tables**\n\nQuery cloud storage to get a list of paths containing CDC files","colour":"e6e63c"}},"variables":{"bucket":{"definition":{"name":"bucket","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"prefix":{"definition":{"name":"prefix","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"account":{"definition":{"name":"account","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"protocol":{"definition":{"name":"protocol","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"container":{"definition":{"name":"container","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"host_path":{"definition":{"name":"host_path","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":null},"cloud_storage_url":{"definition":{"name":"cloud_storage_url","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"azure_key_vault_url":{"definition":{"name":"azure_key_vault_url","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"storage_account_key_secret":{"definition":{"name":"storage_account_key_secret","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""}},"grids":{"source_tables":{"definition":{"name":"source_tables","scope":"TASKBATCH","definitions":[{"name":"source_database","type":"TEXT"},{"name":"source_schema","type":"TEXT"},{"name":"source_table","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]}}}},"transformationJobs":{}}],"version":"1.63.4","environment":"snowflake"}