{"objects":[{"metadata":{"identifier":{"packageName":"Matillion Exchange","name":"Google BigQuery Incremental Load","revision":1,"type":"DYNAMIC"},"rootJobReference":{"name":"Google BigQuery - 1 - Iterate Objects","type":"ORCHESTRATION","parameterMetadata":[{"slot":2,"variableName":"oauth","variableType":"SCALAR","displayName":"OAuth","description":"The name of the OAuth entry to be used from the Matillion OAuth Manager to connect to the source. These must be set up in advance, using <b>Project → Manage OAuth</b>.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":3,"variableName":"source_list","variableType":"GRID","displayName":"Tables and Columns","description":"Contains the list of tables and columns (and in some cases an incremental_column to specify which column the load should be incremented on [accepts values of 0 or 1]) to be processed. ","defaultValue":[{"values":{"1":""}}],"defaultValueType":null,"required":true},{"slot":4,"variableName":"advanced_connection_options","variableType":"GRID","displayName":"Connection Options","description":"A list of values and parameters. Parameters and their allowed values are database/driver specific. Referring to the data model will provide insight of what you could provide here.\n<br>\nThey are usually not required as sensible defaults are assumed.","defaultValue":[{"values":{"1":""}}],"defaultValueType":null,"required":false},{"slot":5,"variableName":"source_dataset","variableType":"SCALAR","displayName":"Source Dataset","description":"The dataset name where the source data is located.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":6,"variableName":"source_project","variableType":"SCALAR","displayName":"Source Project","description":"The project name where the source data is located.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":7,"variableName":"dist_style","variableType":"SCALAR","displayName":"Distribution Style","description":"<b>Even:</b> Distributes rows around the Redshift cluster evenly.<br>\n<b>All:</b> Copy rows to all nodes in the Redshift cluster.<br>\nTable distribution is critical to good performance. See the <a href=\"http://docs.aws.amazon.com/redshift/latest/dg/tutorial-tuning-tables-distribution.html\" target=\"_blank\">Amazon Redshift</a> documentation for more information.The options are:\nEVEN: Distribute data across all nodes evenly.\nALL: Distribute a full copy of the data onto each node.\nNote: KEY distribution is not implemented.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":8,"variableName":"load_concurrent","variableType":"SCALAR","displayName":"Load Type","description":"Sequential - Iterations are done in sequence, waiting for each to complete before starting the next. \nThis is the default.\n<br>\nConcurrent - Iterations are run concurrently. This requires all \"Variables to Iterate\" to be defined as\n<a href=\"https://redshiftsupport.matillion.com/customer/portal/articles/2037630?b_id=8915#header1\" target=\"_blank\"><i>Copied</i> variables</a>, so that each iteration gets its own copy of the variable isolated from the same\nvariable being used by other concurrent executions.\n<span class=\"env rs sf\"><br><b>Note:</b> The maximum concurrency is limited by the number of available threads (2x the number of virtual cpus on your cloud instance).</span>","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":9,"variableName":"s3_bucket_name","variableType":"SCALAR","displayName":"S3 Bucket Name","description":"<span class=\"env sf\">(AWS Only) </span>The name of an S3 bucket for temporary storage. Ensure your access credentials have S3 access and permission to write to the bucket. See <a href=\"https://redshiftsupport.matillion.com/customer/portal/articles/2054760\" target=\"_blank\">this document</a> for details on setting up access. The temporary objects created in this bucket will be removed again after the load completes, they are not kept.<br>\n<span class=\"env sf\">This property is available when using an Existing Amazon S3 Location for Staging.</span>","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":10,"variableName":"stage_schema","variableType":"SCALAR","displayName":"Stage Schema","description":"The schema name where the staging data will be stored.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":11,"variableName":"stage_prefix","variableType":"SCALAR","displayName":"Stage Prefix","description":"A prefix value that will be added to the start of the stage table names.\n<br>\ne.g. If a Stage Prefix of 'stage_' is specified and the table being processed is named 'test_data' then the target table will be named 'stage_test_data'.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":12,"variableName":"target_schema","variableType":"SCALAR","displayName":"Target Schema","description":"The schema name where the target data will be stored.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":13,"variableName":"target_prefix","variableType":"SCALAR","displayName":"Target Prefix","description":"A prefix value that will be added to the start of the target table names.\n<br>\ne.g. If a Target Prefix of 'target_' is specified and the table being processed is named 'test_data' then the target table will be named 'target_test_data'.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":false},{"slot":14,"variableName":"encryption_method","variableType":"SCALAR","displayName":"Encryption","description":"<span class=\"env sf\">(AWS Only) </span>Decide on how the files are encrypted inside the S3 Bucket.<span class=\"env sf\">This property is available when using an Existing Amazon S3 Location for Staging.</span><br>\n                <b>None:</b> No encryption.<br>\n                <b>SSE KMS:</b> Encrypt the data according to a key stored on <a href=\"https://aws.amazon.com/kms/\" target=\"_blank\">KMS</a>.<br>\n                <b>SSE S3:</b> Encrypt the data according to a key stored on an <a href=\"http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html\" target=\"_blank\">S3 bucket</a>","defaultValue":[{"values":{"1":"None"}}],"defaultValueType":"TEXT","required":false},{"slot":15,"variableName":"kms_key_id","variableType":"SCALAR","displayName":"KMS Key ID","description":"The ID of the KMS encryption key you have chosen to use in the 'Encryption' property.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":false},{"slot":16,"variableName":"log_metrics","variableType":"SCALAR","displayName":"Log Metrics","description":"Option to switch off Metrics logging in the full product - default is TRUE which records the metrics for the batch run. ","defaultValue":[{"values":{"1":"TRUE"}}],"defaultValueType":"TEXT","required":false}]},"orchestrationJobs":["Google BigQuery - 1 - Iterate Objects","Google BigQuery - 3 - Stage Object","Google BigQuery - 2 - Stage Object","util - Query Source"],"transformationJobs":[],"imageMetadata":{"componentIcon":"iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAD0ElEQVR42u2Xa2gUVxTHFbWCFBQEFfpJC/VLv4gfNQ+zMY9qGjQG1waKrRqFNgZE2zvjUqdmH3GTaBLU+qjQGkQIilHBfAgitglUfEU0IWpK6mZXk6Dxnc3eOzt/z8xmY4zZ2dkkaAUvHNgd7pz/755z7r1nJkz4OP7vI6MMs2weXpnm5sesGs2vsXlCq8cFwObmDWRI3EJY6hYpYxJPLw3ljk580K7n12LSqMSzqzGVHLSPEQDpbr5hlKEXP41VfMC600sxPSHxVC/m0IvPxgkAVJTlia3ew4+Mm3jEuM3d/4Ul8SUevpBeCJvkFMpJFX+1abj9QMOldg176lVk7oq7K87GVwcm0j5ujOUks5QbwiEB1N8IY3+DitpLYfS+AO4QTF5lnIJ0iUzz0LtCdjMHx5oiYt8dEm88z93N0eLXcLldIM0VjA3h4q0LD2LKiOI5CqbRJF+sl5eVcQQ5UHJKHXzmrFONqOi/C/YLhDVg/b4ecwg33zzyoeMS28xWX/SngEYC2V6OvCqBLMr50z4gp+L1HD0Nu0/3Ikn2I80ZjFVDvfpi3wKg3FebARTXRFaYQSu+/p+G4qPCAFhVxfH7hbAxpzWgofJMLxaze2YQmn6/vF39Jf3zI9tlZIBcWqkIAz8fV40CjAJ8s08YwiupALkKbPqtywDQLVXuwKqdzcPPhBqzi6fCLApnr4Xhewis2SuwYg9H423NKMD8aoG/6XeLL4SkAfEUdhc/SFX4Vj4xNBIv0px9n8U+ARXMoFT0xAJYXs5xs1PDo+cwwr69VjXOgPYuDSpFp6ObI8fpNwA2SgcgSRKYJGOLXIrvd9Tha6e/xMr1u9EsCnrVV597iVv3+vHoGYk/4Pjj/BOs8PjR3NE/ANGJLHYF2yQFG6TDyGfnsFkq0wioS5blL00B9OuThJrj3fXJjte5jtrSX3w433QTq70B478OsYbVReqB/VtAAKcYY22FhYVTzPsAj0i10nAMh/iK/UMh305idwafJbEOLGK+i7pfAphJ9pIg8uKnwiVOJAphZ2co1OXDIxNOZoEFUb8E0EAA3vi9YElwLokEE4GwU7iLpIo3ABbJnYeH+iWAerIKi01JyGW1/9MhclijkYIlrC0K8NQmd8+O+lMU5RMSf0wRKLDWmCj4lDqjgFWIFMd9FEteo/L13CdLga1U9WuHVj4BrCOQaQk0JyKLBB5ahch2tBjb70epUmyVd9r1cJP1xN1+8Ya+Pa0aY8o8Ej1O1qcfRgN2ldxMfKcfNRTqyST8K1mnw+H4/L18WVHo7e9N/IMYrwDqx/c6ukR6/wAAAABJRU5ErkJggg==","smallIcon":"iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAABxElEQVR42mNgoDZwbvup5dL+MxgDt/0Mcuv+L4ZXs0PPfxHnth/vndt//seKW38cZvj/nxGnAU5tP6bh1Aw35GcEVs0uHT91gbb/IWhA249Hln3/ObH4/cdemCKgn/8vOPT3/67Lf//vufrvf96iXyiGOLX/rEd1evvPQGQF3Vv//G/b9Pt/yMSf/907f/7fd+3Pf+/OL8iu+OrW8k0W2fZDyAZsu/j3f+7CX/+XH//zP2Hmr/9923//T5v64r9j8yeEK9p+NiO54EcOsgETd/7+P3v/n//zDv75X7b81/+dl/78d2949N+z4tR/v6ZbIBf8dW39ZYCIvob/LEDBKzAD3Dp+/l927Pf/7Rf+/N998ef/A5e//s+d9eJ/esX0/5mVU/5nVM+9XVJSwo0aiK2/XVFD/Md/+7pX/20qHvx3qrr1v3jCof/2Fbf/u1Rc+pBRNcu9vLy8EzMq235sxDTk5X+vihP/Qyu2gQ2zqXxUBFJbUVGxGDMltv9QAXrlJ7ohznWP/idXzAUZcLOgrFMTqJSxsrIyD3tqbPnhC8wPjSi45XtjYvWKzuKKpr1Ap98DarYnN68xAg2oKCws5KRa7gUAbYiS5dnaydkAAAAASUVORK5CYII="},"description":"","helpHtml":"<div class=\"mtln\"><img src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAD0ElEQVR42u2Xa2gUVxTHFbWCFBQEFfpJC/VLv4gfNQ+zMY9qGjQG1waKrRqFNgZE2zvjUqdmH3GTaBLU+qjQGkQIilHBfAgitglUfEU0IWpK6mZXk6Dxnc3eOzt/z8xmY4zZ2dkkaAUvHNgd7pz/755z7r1nJkz4OP7vI6MMs2weXpnm5sesGs2vsXlCq8cFwObmDWRI3EJY6hYpYxJPLw3ljk580K7n12LSqMSzqzGVHLSPEQDpbr5hlKEXP41VfMC600sxPSHxVC/m0IvPxgkAVJTlia3ew4+Mm3jEuM3d/4Ul8SUevpBeCJvkFMpJFX+1abj9QMOldg176lVk7oq7K87GVwcm0j5ujOUks5QbwiEB1N8IY3+DitpLYfS+AO4QTF5lnIJ0iUzz0LtCdjMHx5oiYt8dEm88z93N0eLXcLldIM0VjA3h4q0LD2LKiOI5CqbRJF+sl5eVcQQ5UHJKHXzmrFONqOi/C/YLhDVg/b4ecwg33zzyoeMS28xWX/SngEYC2V6OvCqBLMr50z4gp+L1HD0Nu0/3Ikn2I80ZjFVDvfpi3wKg3FebARTXRFaYQSu+/p+G4qPCAFhVxfH7hbAxpzWgofJMLxaze2YQmn6/vF39Jf3zI9tlZIBcWqkIAz8fV40CjAJ8s08YwiupALkKbPqtywDQLVXuwKqdzcPPhBqzi6fCLApnr4Xhewis2SuwYg9H423NKMD8aoG/6XeLL4SkAfEUdhc/SFX4Vj4xNBIv0px9n8U+ARXMoFT0xAJYXs5xs1PDo+cwwr69VjXOgPYuDSpFp6ObI8fpNwA2SgcgSRKYJGOLXIrvd9Tha6e/xMr1u9EsCnrVV597iVv3+vHoGYk/4Pjj/BOs8PjR3NE/ANGJLHYF2yQFG6TDyGfnsFkq0wioS5blL00B9OuThJrj3fXJjte5jtrSX3w433QTq70B478OsYbVReqB/VtAAKcYY22FhYVTzPsAj0i10nAMh/iK/UMh305idwafJbEOLGK+i7pfAphJ9pIg8uKnwiVOJAphZ2co1OXDIxNOZoEFUb8E0EAA3vi9YElwLokEE4GwU7iLpIo3ABbJnYeH+iWAerIKi01JyGW1/9MhclijkYIlrC0K8NQmd8+O+lMU5RMSf0wRKLDWmCj4lDqjgFWIFMd9FEteo/L13CdLga1U9WuHVj4BrCOQaQk0JyKLBB5ahch2tBjb70epUmyVd9r1cJP1xN1+8Ya+Pa0aY8o8Ej1O1qcfRgN2ldxMfKcfNRTqyST8K1mnw+H4/L18WVHo7e9N/IMYrwDqx/c6ukR6/wAAAABJRU5ErkJggg==\"><h3>Google BigQuery Incremental Load</h3><p style='white-space: pre-line; word-wrap: break-word;'></p><h3>Properties</h3><table><tbody><tr><th>Property</th><th>Type</th><th>Description</th></tr><tr><td>OAuth</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>The name of the OAuth entry to be used from the Matillion OAuth Manager to connect to the source. These must be set up in advance, using <b>Project → Manage OAuth</b>.</td></tr><tr><td>Tables and Columns</td><td>Grid</td><td style='white-space: pre-line; word-wrap: break-word;'>Contains the list of tables and columns (and in some cases an incremental_column to specify which column the load should be incremented on [accepts values of 0 or 1]) to be processed. </td></tr><tr><td>Connection Options</td><td>Grid</td><td style='white-space: pre-line; word-wrap: break-word;'>A list of values and parameters. Parameters and their allowed values are database/driver specific. Referring to the data model will provide insight of what you could provide here.\n<br>\nThey are usually not required as sensible defaults are assumed.</td></tr><tr><td>Source Dataset</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>The dataset name where the source data is located.</td></tr><tr><td>Source Project</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>The project name where the source data is located.</td></tr><tr><td>Distribution Style</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'><b>Even:</b> Distributes rows around the Redshift cluster evenly.<br>\n<b>All:</b> Copy rows to all nodes in the Redshift cluster.<br>\nTable distribution is critical to good performance. See the <a href=\"http://docs.aws.amazon.com/redshift/latest/dg/tutorial-tuning-tables-distribution.html\" target=\"_blank\">Amazon Redshift</a> documentation for more information.The options are:\nEVEN: Distribute data across all nodes evenly.\nALL: Distribute a full copy of the data onto each node.\nNote: KEY distribution is not implemented.</td></tr><tr><td>Load Type</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>Sequential - Iterations are done in sequence, waiting for each to complete before starting the next. \nThis is the default.\n<br>\nConcurrent - Iterations are run concurrently. This requires all \"Variables to Iterate\" to be defined as\n<a href=\"https://redshiftsupport.matillion.com/customer/portal/articles/2037630?b_id=8915#header1\" target=\"_blank\"><i>Copied</i> variables</a>, so that each iteration gets its own copy of the variable isolated from the same\nvariable being used by other concurrent executions.\n<span class=\"env rs sf\"><br><b>Note:</b> The maximum concurrency is limited by the number of available threads (2x the number of virtual cpus on your cloud instance).</span></td></tr><tr><td>S3 Bucket Name</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'><span class=\"env sf\">(AWS Only) </span>The name of an S3 bucket for temporary storage. Ensure your access credentials have S3 access and permission to write to the bucket. See <a href=\"https://redshiftsupport.matillion.com/customer/portal/articles/2054760\" target=\"_blank\">this document</a> for details on setting up access. The temporary objects created in this bucket will be removed again after the load completes, they are not kept.<br>\n<span class=\"env sf\">This property is available when using an Existing Amazon S3 Location for Staging.</span></td></tr><tr><td>Stage Schema</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>The schema name where the staging data will be stored.</td></tr><tr><td>Stage Prefix</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>A prefix value that will be added to the start of the stage table names.\n<br>\ne.g. If a Stage Prefix of 'stage_' is specified and the table being processed is named 'test_data' then the target table will be named 'stage_test_data'.</td></tr><tr><td>Target Schema</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>The schema name where the target data will be stored.</td></tr><tr><td>Target Prefix</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>A prefix value that will be added to the start of the target table names.\n<br>\ne.g. If a Target Prefix of 'target_' is specified and the table being processed is named 'test_data' then the target table will be named 'target_test_data'.</td></tr><tr><td>Encryption</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'><span class=\"env sf\">(AWS Only) </span>Decide on how the files are encrypted inside the S3 Bucket.<span class=\"env sf\">This property is available when using an Existing Amazon S3 Location for Staging.</span><br>\n                <b>None:</b> No encryption.<br>\n                <b>SSE KMS:</b> Encrypt the data according to a key stored on <a href=\"https://aws.amazon.com/kms/\" target=\"_blank\">KMS</a>.<br>\n                <b>SSE S3:</b> Encrypt the data according to a key stored on an <a href=\"http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html\" target=\"_blank\">S3 bucket</a></td></tr><tr><td>KMS Key ID</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>The ID of the KMS encryption key you have chosen to use in the 'Encryption' property.</td></tr><tr><td>Log Metrics</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>Option to switch off Metrics logging in the full product - default is TRUE which records the metrics for the batch run. </td></tr></tbody></table></div>","created":1607455214393},"orchestrationJobs":{"util - Query Source":{"id":-2,"revision":1,"created":1607422213591,"timestamp":1607422213591,"components":{"35994":{"id":35994,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186960,"x":3,"y":-2,"width":32,"height":32,"inputConnectorIDs":[35995],"outputSuccessConnectorIDs":[35998],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"msg = \\\nf\"\"\"\nTable: {table}\nSQL:\n{sql}\n\"\"\"\n\nprint(msg)"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"37851":{"id":37851,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-888888192,"x":160,"y":0,"width":32,"height":32,"inputConnectorIDs":[35998],"outputSuccessConnectorIDs":[38468],"outputFailureConnectorIDs":[38722],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"Message","mapTo":"queryErrorMessage","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"Status","mapTo":"queryExecutionStatus","validationStatus":null,"validationMessage":null},"3":{"slot":3,"fromId":null,"fromName":"Row Count","mapTo":"queryRowCount","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Execute SQL"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Basic/Advanced Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Advanced"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Authentication","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${oauth}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Data Source","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Data Selection","elements":{},"visible":false,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Data Source Filter","elements":{},"visible":false,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Combine Filters","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":false,"status":{},"columnNames":[]},"10":{"slot":10,"name":"SQL Query","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${sql}"}}}},"visible":true,"status":{},"columnNames":[]},"11":{"slot":11,"name":"Limit","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"100"}}}},"visible":false,"status":{},"columnNames":[]},"12":{"slot":12,"name":"Connection Options","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"ConnectionOptions"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"parameter"},"2":{"slot":2,"type":"GRID","value":"value"}}}},"visible":true,"status":{},"columnNames":[]},"13":{"slot":13,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${table}"}}}},"visible":true,"status":{},"columnNames":[]},"14":{"slot":14,"name":"Distribution Style","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${dist_style}"}}}},"visible":true,"status":{},"columnNames":[]},"15":{"slot":15,"name":"Distribution Key","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"16":{"slot":16,"name":"Sort Key","elements":{},"visible":true,"status":{},"columnNames":[]},"17":{"slot":17,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${schema}"}}}},"visible":true,"status":{},"columnNames":[]},"18":{"slot":18,"name":"Sort Key Options","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"19":{"slot":19,"name":"S3 Staging Area","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${bucket_name}"}}}},"visible":true,"status":{},"columnNames":[]},"20":{"slot":20,"name":"Data Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"21":{"slot":21,"name":"Primary Key","elements":{},"visible":true,"status":{},"columnNames":[]},"95":{"slot":95,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"96":{"slot":96,"name":"Location","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"s3://<bucket>/<path>"}}}},"visible":false,"status":{},"columnNames":[]},"97":{"slot":97,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"98":{"slot":98,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"99":{"slot":99,"name":"Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Standard"}}}},"visible":true,"status":{},"columnNames":[]},"106":{"slot":106,"name":"Project ID","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${source_project}"}}}},"visible":true,"status":{},"columnNames":[]},"107":{"slot":107,"name":"Dataset ID","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${source_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"108":{"slot":108,"name":"SQL Dialect","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${z_bq_dialect}"}}}},"visible":true,"status":{},"columnNames":[]},"1001":{"slot":1001,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"1013":{"slot":1013,"name":"Auto Debug","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${auto_debug}"}}}},"visible":true,"status":{},"columnNames":[]},"1014":{"slot":1014,"name":"Debug Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${auto_debug_level}"}}}},"visible":false,"status":{},"columnNames":[]},"1034":{"slot":1034,"name":"Load Options","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"LoadOptions"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"parameter"},"2":{"slot":2,"type":"GRID","value":"value"}}}},"visible":true,"status":{},"columnNames":[]},"40000":{"slot":40000,"name":"Encryption","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${encryption_method}"}}}},"visible":true,"status":{},"columnNames":[]},"40001":{"slot":40001,"name":"KMS Key ID","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${kms_key_id}"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"37893":{"id":37893,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":-160,"y":0,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[35995],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Begin"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"38467":{"id":38467,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":320,"y":0,"width":32,"height":32,"inputConnectorIDs":[38468],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"38625":{"id":38625,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186960,"x":160,"y":80,"width":32,"height":32,"inputConnectorIDs":[38722],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[38789],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Print Error"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"msg = \\\nf\"\"\"{queryExecutionStatus}: Unable to retrieve data from the source.\n\nThe attempted SQL was:\n{sql}\n \nError: {queryErrorMessage}\n\"\"\"\n\nprint(msg)"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"38733":{"id":38733,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":160,"y":160,"width":32,"height":32,"inputConnectorIDs":[38789],"outputSuccessConnectorIDs":[38740],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Error"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"error"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${ctx}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Unable to retrieve data from source"},"2":{"slot":2,"type":"STRING","value":"${queryErrorMessage}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"38739":{"id":38739,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":160,"y":240,"width":32,"height":32,"inputConnectorIDs":[38740],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End: Error"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"35998":{"id":35998,"sourceID":35994,"targetID":37851},"38468":{"id":38468,"sourceID":37851,"targetID":38467},"38740":{"id":38740,"sourceID":38733,"targetID":38739}},"failureConnectors":{"38722":{"id":38722,"sourceID":37851,"targetID":38625}},"unconditionalConnectors":{"35995":{"id":35995,"sourceID":37893,"targetID":35994},"38789":{"id":38789,"sourceID":38625,"targetID":38733}},"trueConnectors":{},"falseConnectors":{},"iterationConnectors":{},"noteConnectors":{},"canUndo":false,"undoCommand":"","undoCreated":0,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{},"variables":{"ctx":{"definition":{"name":"ctx","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"sql":{"definition":{"name":"sql","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"SELECT 1"},"oauth":{"definition":{"name":"oauth","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"Google"},"table":{"definition":{"name":"table","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"mockTableName"},"schema":{"definition":{"name":"schema","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"server":{"definition":{"name":"server","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"mock.Server"},"dataset":{"definition":{"name":"dataset","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"project":{"definition":{"name":"project","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environmetn Default]"},"database":{"definition":{"name":"database","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"password":{"definition":{"name":"password","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"mockPassword"},"username":{"definition":{"name":"username","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"mockUsername"},"warehouse":{"definition":{"name":"warehouse","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"auto_debug":{"definition":{"name":"auto_debug","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"Off"},"dist_style":{"definition":{"name":"dist_style","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"Even"},"kms_key_id":{"definition":{"name":"kms_key_id","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"bucket_name":{"definition":{"name":"bucket_name","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"staging_type":{"definition":{"name":"staging_type","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"Snowflake Managed"},"z_bq_dialect":{"definition":{"name":"z_bq_dialect","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"Standard"},"queryRowCount":{"definition":{"name":"queryRowCount","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"},"blob_container":{"definition":{"name":"blob_container","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"source_dataset":{"definition":{"name":"source_dataset","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"source_project":{"definition":{"name":"source_project","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"storage_account":{"definition":{"name":"storage_account","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"auto_debug_level":{"definition":{"name":"auto_debug_level","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"3"},"encryption_method":{"definition":{"name":"encryption_method","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"None"},"queryErrorMessage":{"definition":{"name":"queryErrorMessage","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"cloud_storage_area":{"definition":{"name":"cloud_storage_area","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"queryExecutionStatus":{"definition":{"name":"queryExecutionStatus","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""}},"grids":{"LoadOptions":{"definition":{"name":"LoadOptions","scope":"BRANCH","definitions":[{"name":"parameter","type":"TEXT"},{"name":"value","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]},"ConnectionOptions":{"definition":{"name":"ConnectionOptions","scope":"BRANCH","definitions":[{"name":"parameter","type":"TEXT"},{"name":"value","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]}}},"Google BigQuery - 1 - Iterate Objects":{"id":-2,"revision":1,"created":1607422213585,"timestamp":1607422213585,"components":{"123":{"id":123,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-2112089365,"x":160,"y":-80,"width":32,"height":32,"inputConnectorIDs":[13651],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Duration"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Metrics","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"duration"},"2":{"slot":2,"type":"STRING","value":"Numeric"},"3":{"slot":3,"type":"STRING","value":"${logging_duration}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":" "}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"13635":{"id":13635,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":0,"y":-80,"width":32,"height":32,"inputConnectorIDs":[13636],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[13651],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Metrics?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"log_metrics"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"TRUE"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"21429":{"id":21429,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":0,"y":32,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"GBQ Stage Object"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Google BigQuery - 2 - Stage Object"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"bucket_name"},"2":{"slot":2,"type":"STRING","value":"${s3_bucket_name}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"stage_prefix"},"2":{"slot":2,"type":"STRING","value":"${stage_prefix}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"stage_schema"},"2":{"slot":2,"type":"STRING","value":"${stage_schema}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"target_prefix"},"2":{"slot":2,"type":"STRING","value":"${target_prefix}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"target_schema"},"2":{"slot":2,"type":"STRING","value":"${target_schema}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"table_name"},"2":{"slot":2,"type":"STRING","value":"${table_name}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"dist_style"},"2":{"slot":2,"type":"STRING","value":"${dist_style}"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"connection_url"},"2":{"slot":2,"type":"STRING","value":"${connection_url}"}}},"9":{"slot":9,"values":{"1":{"slot":1,"type":"STRING","value":"target_platform"},"2":{"slot":2,"type":"STRING","value":"${target_platform}"}}},"10":{"slot":10,"values":{"1":{"slot":1,"type":"STRING","value":"target_database"},"2":{"slot":2,"type":"STRING","value":"${target_database}"}}},"11":{"slot":11,"values":{"1":{"slot":1,"type":"STRING","value":"target_warehouse"},"2":{"slot":2,"type":"STRING","value":"${target_warehouse}"}}},"12":{"slot":12,"values":{"1":{"slot":1,"type":"STRING","value":"stage_database"},"2":{"slot":2,"type":"STRING","value":"${stage_database}"}}},"13":{"slot":13,"values":{"1":{"slot":1,"type":"STRING","value":"stage_warehouse"},"2":{"slot":2,"type":"STRING","value":"${stage_warehouse}"}}},"14":{"slot":14,"values":{"1":{"slot":1,"type":"STRING","value":"target_dataset"},"2":{"slot":2,"type":"STRING","value":"${target_dataset}"}}},"15":{"slot":15,"values":{"1":{"slot":1,"type":"STRING","value":"target_project"},"2":{"slot":2,"type":"STRING","value":"${target_project}"}}},"16":{"slot":16,"values":{"1":{"slot":1,"type":"STRING","value":"stage_dataset"},"2":{"slot":2,"type":"STRING","value":"${stage_dataset}"}}},"17":{"slot":17,"values":{"1":{"slot":1,"type":"STRING","value":"stage_project"},"2":{"slot":2,"type":"STRING","value":"${stage_project}"}}},"18":{"slot":18,"values":{"1":{"slot":1,"type":"STRING","value":"cloud_storage_area"},"2":{"slot":2,"type":"STRING","value":"${cloud_storage_area}"}}},"19":{"slot":19,"values":{"1":{"slot":1,"type":"STRING","value":"storage_account"},"2":{"slot":2,"type":"STRING","value":"${storage_account}"}}},"20":{"slot":20,"values":{"1":{"slot":1,"type":"STRING","value":"blob_container"},"2":{"slot":2,"type":"STRING","value":"${blob_container}"}}},"21":{"slot":21,"values":{"1":{"slot":1,"type":"STRING","value":"oauth"},"2":{"slot":2,"type":"STRING","value":"${oauth}"}}},"22":{"slot":22,"values":{"1":{"slot":1,"type":"STRING","value":"source_dataset"},"2":{"slot":2,"type":"STRING","value":"${source_dataset}"}}},"23":{"slot":23,"values":{"1":{"slot":1,"type":"STRING","value":"source_project"},"2":{"slot":2,"type":"STRING","value":"${source_project}"}}},"24":{"slot":24,"values":{"1":{"slot":1,"type":"STRING","value":"encryption_method"},"2":{"slot":2,"type":"STRING","value":"${encryption_method}"}}},"25":{"slot":25,"values":{"1":{"slot":1,"type":"STRING","value":"kms_key_id"},"2":{"slot":2,"type":"STRING","value":"${kms_key_id}"}}},"26":{"slot":26,"values":{"1":{"slot":1,"type":"STRING","value":"staging_type"},"2":{"slot":2,"type":"STRING","value":"${staging_type}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"advanced_connection_options"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"parameter"},"2":{"slot":2,"type":"GRID","value":"value"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"advanced_connection_options"},"2":{"slot":2,"type":"STRING","value":"grid"}}},"2":{"slot":2,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"source_list"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"table"},"2":{"slot":2,"type":"GRID","value":"column"},"3":{"slot":3,"type":"GRID","value":"incremental_column"},"4":{"slot":4,"type":"GRID","value":"key_column"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"source_list"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Google BigQuery Incremental Load"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Incremental"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[21441]},"21430":{"id":21430,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186960,"x":-160,"y":0,"width":32,"height":32,"inputConnectorIDs":[21436],"outputSuccessConnectorIDs":[21438],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Update Environment Default"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"# Script to replace any [Environment Default] entries with the schema name\n\nif target_platform == 'Redshift':\n  \n    if stage_schema == '[Environment Default]':\n        context.updateVariable('stage_schema', environment_default_schema)\n  \n    if target_schema == '[Environment Default]':\n        context.updateVariable('target_schema', environment_default_schema)\n\nelif target_platform == 'Snowflake':\n    \n    if stage_schema == '[Environment Default]':\n        context.updateVariable('stage_schema', environment_default_schema)\n  \n    if target_schema == '[Environment Default]':\n        context.updateVariable('target_schema', environment_default_schema)\n\n    if stage_database == '[Environment Default]':\n        context.updateVariable('stage_database', environment_database)\n\n    if target_database == '[Environment Default]':\n        context.updateVariable('target_database', environment_database)\n  \nelif target_platform == 'BigQuery':\n  \n    if stage_dataset == '[Environment Default]':\n        context.updateVariable('stage_dataset', environment_default_schema)\n\n    if target_dataset == '[Environment Default]':\n        context.updateVariable('target_dataset', environment_default_schema)\n\n    if stage_project == '[Environment Default]':\n        context.updateVariable('stage_project', environment_database)\n\n    if target_project == '[Environment Default]':\n        context.updateVariable('target_project', environment_database)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"21431":{"id":21431,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186960,"x":-320,"y":0,"width":32,"height":32,"inputConnectorIDs":[21439],"outputSuccessConnectorIDs":[21436],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Get unique table names"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"source_list = context.getGridVariable('source_list')\n\n# get distinct list of table names from source_list\ndistinct_table_names = list(set([table for table, col, inc_col, key_col in source_list]))\n\n# convert to list of lists to store in grid variable\ndistinct_table_names = [[name] for name in sorted(distinct_table_names)]\ncontext.updateGridVariable(\"distinct_table_names\", distinct_table_names)\n\nprint(f\"source_list: {str(source_list)}\")\nprint(f\"distinct_table_names: {str(distinct_table_names)}\")\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"21432":{"id":21432,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":160,"y":80,"width":32,"height":32,"inputConnectorIDs":[21440],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Failure"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"21433":{"id":21433,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":160,"y":0,"width":32,"height":32,"inputConnectorIDs":[21437],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Success"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"21434":{"id":21434,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":-480,"y":0,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[21439],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"21435":{"id":21435,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"ITERATE","implementationID":-424773870,"x":0,"y":0,"width":32,"height":16,"inputConnectorIDs":[21438],"outputSuccessConnectorIDs":[13636,21437],"outputFailureConnectorIDs":[21440],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"Duration","mapTo":"logging_duration","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Iterate over distinct table names"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Grid Variable","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"distinct_table_names"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Grid Variable Mapping","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"table_name"},"2":{"slot":2,"type":"STRING","value":"table_name"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Break on Failure","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"No"}}}},"visible":false,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Concurrency","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${load_concurrent}"}}}},"visible":true,"status":{},"columnNames":[]},"999":{"slot":999,"name":"Record Values In Task History","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Yes"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[21441],"inputIterationConnectorIDs":[]}},"successConnectors":{"13636":{"id":13636,"sourceID":21435,"targetID":13635},"21436":{"id":21436,"sourceID":21431,"targetID":21430},"21437":{"id":21437,"sourceID":21435,"targetID":21433},"21438":{"id":21438,"sourceID":21430,"targetID":21435}},"failureConnectors":{"21440":{"id":21440,"sourceID":21435,"targetID":21432}},"unconditionalConnectors":{"21439":{"id":21439,"sourceID":21434,"targetID":21431}},"trueConnectors":{"13651":{"id":13651,"sourceID":13635,"targetID":123}},"falseConnectors":{},"iterationConnectors":{"21441":{"id":21441,"sourceID":21435,"targetID":21429}},"noteConnectors":{},"canUndo":false,"undoCommand":"","undoCreated":0,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"21427":{"id":21427,"x":-639,"y":-503,"width":1110,"height":390,"text":"**Google BigQuery load into cloud data warehouse ** \nTop level orchestration process which reads the list of table names and column names provided via UI, then iterates \nthrough the table name list to process loading of the supplied columns.\n\nParameters are set from the calling UI as part of the Matillion shared job.\n\n======================================================================\n**Predecessor Information:**\nNone\n\n======================================================================\n**Successor Information:**\nGoogle BigQuery - 2 - Stage Object\n\n======================================================================\n**Change History - __please append new changes as the first entry below ensuring date, author, version and change detail is recorded __**\n----------------------------------------------------------------------------------------------------------------------------\nVersion 1.0\nDate: Aug 2019\nAuthor: Matillion ELT\nChange Summary: Initial version of GBQ load into cloud data warehouse.\nChange Detail: Initial version\n\n__**Change detail should be documented on the canvas or within this note for traceability.**__\n======================================================================\n**Nov 2020**:\nAdjust job for newer pattern - introduce intermediary Stage 2 job to wrap the original stage 2 job","colour":"00ce4f"}},"variables":{"oauth":{"definition":{"name":"oauth","type":"TEXT","scope":"TASKBATCH","description":"The name of the OAuth entry to be used from the Matillion OAuth Manager to connect to the source. These must be set up in advance, using <b>Project → Manage OAuth</b>.","visibility":"PUBLIC"},"value":""},"user_name":{"definition":{"name":"user_name","type":"TEXT","scope":"BRANCH","description":"User name to use to connect to the MongoDB cluster","visibility":"PRIVATE"},"value":""},"dist_style":{"definition":{"name":"dist_style","type":"TEXT","scope":"BRANCH","description":"The distribution style to be used for both the staged and target data.\nThe options are:\nEVEN: Distribute data across all nodes evenly.\nALL: Distribute a full copy of the data onto each node.\nNote: KEY distribution is not implemented.","visibility":"PUBLIC"},"value":""},"kms_key_id":{"definition":{"name":"kms_key_id","type":"TEXT","scope":"BRANCH","description":"The ID of the KMS encryption key you have chosen to use in the 'Encryption' property.","visibility":"PUBLIC"},"value":""},"table_name":{"definition":{"name":"table_name","type":"TEXT","scope":"BRANCH","description":"","visibility":"PRIVATE"},"value":""},"log_metrics":{"definition":{"name":"log_metrics","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"TRUE"},"stage_prefix":{"definition":{"name":"stage_prefix","type":"TEXT","scope":"BRANCH","description":"The prefix to assign to each of the source tables when they are staged. This is user assigned to help avoid any potential clashes with existing tables.\nE.g. If stage_prefix is set to 'stage_' and the table being processed is named 'test_data' then the staged table will be stored as 'stage_test_data'.","visibility":"PUBLIC"},"value":""},"stage_schema":{"definition":{"name":"stage_schema","type":"TEXT","scope":"BRANCH","description":"The schema where the staged data will be stored.","visibility":"PUBLIC"},"value":""},"staging_type":{"definition":{"name":"staging_type","type":"TEXT","scope":"BRANCH","description":"(AWS Only) <b>Snowflake Managed:</b> Allow Matillion ETL to create and use a temporary internal stage on Snowflake for staging the data. This stage, along with the staged data, will cease to exist after loading is complete.<br>\n                <b>Existing Amazon S3 Location:</b> Selecting this will avail the user of properties to specify a custom staging area on S3.","visibility":"PRIVATE"},"value":"Snowflake Managed"},"stage_dataset":{"definition":{"name":"stage_dataset","type":"TEXT","scope":"TASKBATCH","description":"The GCP dataset where the staged data will be stored.","visibility":"PRIVATE"},"value":""},"stage_project":{"definition":{"name":"stage_project","type":"TEXT","scope":"TASKBATCH","description":"The GCP project where the staged data will be stored.","visibility":"PRIVATE"},"value":""},"target_prefix":{"definition":{"name":"target_prefix","type":"TEXT","scope":"BRANCH","description":"The prefix to assign to each of the target tables when they are created/updated. This is user assigned to help avoid any potential clashes with existing tables.\nE.g. If target_prefix is set to 'target_' and the table being processed is named 'test_data' then the staged table will be stored as 'target_test_data'.","visibility":"PUBLIC"},"value":""},"target_schema":{"definition":{"name":"target_schema","type":"TEXT","scope":"BRANCH","description":"The schema where the target data will be stored.","visibility":"PUBLIC"},"value":""},"blob_container":{"definition":{"name":"blob_container","type":"TEXT","scope":"BRANCH","description":"","visibility":"PRIVATE"},"value":""},"connection_url":{"definition":{"name":"connection_url","type":"TEXT","scope":"BRANCH","description":"Hostname or IP address of the MongoDB cluster","visibility":"PRIVATE"},"value":""},"s3_bucket_name":{"definition":{"name":"s3_bucket_name","type":"TEXT","scope":"BRANCH","description":"<span class=\"env sf\">(AWS Only) </span>The name of an S3 bucket for temporary storage. Ensure your access credentials have S3 access and permission to write to the bucket. See <a href=\"https://redshiftsupport.matillion.com/customer/portal/articles/2054760\" target=\"_blank\">this document</a> for details on setting up access. The temporary objects created in this bucket will be removed again after the load completes, they are not kept.<br>\n<span class=\"env sf\">This property is available when using an Existing Amazon S3 Location for Staging.</span>","visibility":"PUBLIC"},"value":""},"source_dataset":{"definition":{"name":"source_dataset","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"source_project":{"definition":{"name":"source_project","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"stage_database":{"definition":{"name":"stage_database","type":"TEXT","scope":"BRANCH","description":"The Snowflake database where the staged data will be stored.","visibility":"PRIVATE"},"value":""},"target_dataset":{"definition":{"name":"target_dataset","type":"TEXT","scope":"TASKBATCH","description":"The GCP dataset where the target data will be stored.","visibility":"PRIVATE"},"value":""},"target_project":{"definition":{"name":"target_project","type":"TEXT","scope":"TASKBATCH","description":"The GCP project where the target data will be stored.","visibility":"PRIVATE"},"value":""},"load_concurrent":{"definition":{"name":"load_concurrent","type":"TEXT","scope":"BRANCH","description":"Sequential - Iterations are done in sequence, waiting for each to complete before starting the next. \nThis is the default.\n<br>\nConcurrent - Iterations are run concurrently. This requires all \"Variables to Iterate\" to be defined as\n<a href=\"https://redshiftsupport.matillion.com/customer/portal/articles/2037630?b_id=8915#header1\" target=\"_blank\"><i>Copied</i> variables</a>, so that each iteration gets its own copy of the variable isolated from the same\nvariable being used by other concurrent executions.\n<span class=\"env rs sf\"><br><b>Note:</b> The maximum concurrency is limited by the number of available threads (2x the number of virtual cpus on your cloud instance).</span>","visibility":"PUBLIC"},"value":""},"stage_warehouse":{"definition":{"name":"stage_warehouse","type":"TEXT","scope":"BRANCH","description":"The Snowflake warehouse where the stage data will be stored.","visibility":"PRIVATE"},"value":""},"storage_account":{"definition":{"name":"storage_account","type":"TEXT","scope":"BRANCH","description":"","visibility":"PRIVATE"},"value":""},"target_database":{"definition":{"name":"target_database","type":"TEXT","scope":"BRANCH","description":"The Snowflake database where the target data will be stored.","visibility":"PRIVATE"},"value":""},"target_platform":{"definition":{"name":"target_platform","type":"TEXT","scope":"BRANCH","description":"######( Internal Use Only )######\n Used to help with cross platform compatibility.","visibility":"PRIVATE"},"value":"Redshift"},"logging_duration":{"definition":{"name":"logging_duration","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"target_warehouse":{"definition":{"name":"target_warehouse","type":"TEXT","scope":"BRANCH","description":"The Snowflake warehouse where the target data will be stored.","visibility":"PRIVATE"},"value":""},"encryption_method":{"definition":{"name":"encryption_method","type":"TEXT","scope":"BRANCH","description":"<span class=\"env sf\">(AWS Only) </span>Decide on how the files are encrypted inside the S3 Bucket.<span class=\"env sf\">This property is available when using an Existing Amazon S3 Location for Staging.</span><br>\n                <b>None:</b> No encryption.<br>\n                <b>SSE KMS:</b> Encrypt the data according to a key stored on <a href=\"https://aws.amazon.com/kms/\" target=\"_blank\">KMS</a>.<br>\n                <b>SSE S3:</b> Encrypt the data according to a key stored on an <a href=\"http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html\" target=\"_blank\">S3 bucket</a>","visibility":"PUBLIC"},"value":"None"},"cloud_storage_area":{"definition":{"name":"cloud_storage_area","type":"TEXT","scope":"BRANCH","description":"The cloud storage area name that should be used when loading bq tables.","visibility":"PRIVATE"},"value":""},"logged_rows_deleted":{"definition":{"name":"logged_rows_deleted","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"logged_rows_updated":{"definition":{"name":"logged_rows_updated","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"logged_rows_inserted":{"definition":{"name":"logged_rows_inserted","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"password_manager_entry":{"definition":{"name":"password_manager_entry","type":"TEXT","scope":"BRANCH","description":"Name of the password manager entry which contains the password to use when connecting to the MongoDB cluster","visibility":"PRIVATE"},"value":""}},"grids":{"source_list":{"definition":{"name":"source_list","scope":"BRANCH","definitions":[{"name":"table","type":"TEXT"},{"name":"column","type":"TEXT"},{"name":"incremental_column","type":"DECIMAL"},{"name":"key_column","type":"DECIMAL"}],"description":"Contains the list of tables and columns to be processed. The incremental_column should be set to 1 to indicate which column the incremental load is to be based on, 0 otherwise. There should be only 1 incremental column per table","visibility":"PUBLIC"},"values":[]},"distinct_table_names":{"definition":{"name":"distinct_table_names","scope":"BRANCH","definitions":[{"name":"table_name","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[]},"advanced_connection_options":{"definition":{"name":"advanced_connection_options","scope":"BRANCH","definitions":[{"name":"parameter","type":"TEXT"},{"name":"value","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]}}},"Google BigQuery - 3 - Stage Object":{"id":-2,"revision":8,"created":1607429051310,"timestamp":1607429051310,"components":{"39399":{"id":39399,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":-560,"y":-80,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[36009],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Begin"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"39400":{"id":39400,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":1010676649,"x":-400,"y":-80,"width":32,"height":32,"inputConnectorIDs":[36009],"outputSuccessConnectorIDs":[36013],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"source_metadata_sql","mapTo":"source_metadata_sql","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"get_metadata_sql","mapTo":"get_metadata_sql","validationStatus":null,"validationMessage":null},"3":{"slot":3,"fromId":null,"fromName":"target_table_name","mapTo":"target_table_name","validationStatus":null,"validationMessage":null},"4":{"slot":4,"fromId":null,"fromName":"stage_table_name","mapTo":"stage_table_name","validationStatus":null,"validationMessage":null},"5":{"slot":5,"fromId":null,"fromName":"metadata_stage_table_name","mapTo":"metadata_stage_table_name","validationStatus":null,"validationMessage":null}},"gridExportMappings":{"1":{"slot":1,"fromGrid":"source_list","toGrid":"source_list","columnMapToFrom":{"column":"column","key_column":"key_column","incremental_column":"incremental_column","table":"table"},"validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Build strings"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"stage_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_database}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"stage_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"stage_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_project}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"stage_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"stage_prefix","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_prefix}"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"target_prefix","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_prefix}"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"metadata_stage_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"10":{"slot":10,"name":"stage_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"11":{"slot":11,"name":"target_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"12":{"slot":12,"name":"source_metadata_sql","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"13":{"slot":13,"name":"get_metadata_sql","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"14":{"slot":14,"name":"source_list","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"source_list"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"table"},"2":{"slot":2,"type":"GRID","value":"column"},"3":{"slot":3,"type":"GRID","value":"incremental_column"},"4":{"slot":4,"type":"GRID","value":"key_column"}}}},"visible":true,"status":{},"columnNames":[]},"15":{"slot":15,"name":"random_suffix","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"39424":{"id":39424,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":-240,"y":-80,"width":32,"height":32,"inputConnectorIDs":[36013],"outputSuccessConnectorIDs":[39429],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Get Source Metadata"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"util - Query Source"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"ctx"},"2":{"slot":2,"type":"STRING","value":"${target_table_name}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"sql"},"2":{"slot":2,"type":"STRING","value":"${source_metadata_sql}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"warehouse"},"2":{"slot":2,"type":"STRING","value":"${stage_warehouse}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"database"},"2":{"slot":2,"type":"STRING","value":"${stage_database}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"schema"},"2":{"slot":2,"type":"STRING","value":"${stage_schema}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"project"},"2":{"slot":2,"type":"STRING","value":"${stage_project}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"dataset"},"2":{"slot":2,"type":"STRING","value":"${stage_dataset}"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"table"},"2":{"slot":2,"type":"STRING","value":"${metadata_stage_table_name}"}}},"9":{"slot":9,"values":{"1":{"slot":1,"type":"STRING","value":"bucket_name"},"2":{"slot":2,"type":"STRING","value":"${bucket_name}"}}},"10":{"slot":10,"values":{"1":{"slot":1,"type":"STRING","value":"dist_style"},"2":{"slot":2,"type":"STRING","value":"${dist_style}"}}},"11":{"slot":11,"values":{"1":{"slot":1,"type":"STRING","value":"encryption_method"},"2":{"slot":2,"type":"STRING","value":"${encryption_method}"}}},"12":{"slot":12,"values":{"1":{"slot":1,"type":"STRING","value":"kms_key_id"},"2":{"slot":2,"type":"STRING","value":"${kms_key_id}"}}},"13":{"slot":13,"values":{"1":{"slot":1,"type":"STRING","value":"staging_type"},"2":{"slot":2,"type":"STRING","value":"${staging_type}"}}},"14":{"slot":14,"values":{"1":{"slot":1,"type":"STRING","value":"storage_account"},"2":{"slot":2,"type":"STRING","value":"${storage_account}"}}},"15":{"slot":15,"values":{"1":{"slot":1,"type":"STRING","value":"blob_container"},"2":{"slot":2,"type":"STRING","value":"${blob_container}"}}},"16":{"slot":16,"values":{"1":{"slot":1,"type":"STRING","value":"cloud_storage_area"},"2":{"slot":2,"type":"STRING","value":"${cloud_storage_area}"}}},"17":{"slot":17,"values":{"1":{"slot":1,"type":"STRING","value":"oauth"},"2":{"slot":2,"type":"STRING","value":"${oauth}"}}},"18":{"slot":18,"values":{"1":{"slot":1,"type":"STRING","value":"z_bq_dialect"},"2":{"slot":2,"type":"STRING","value":"Standard"}}},"19":{"slot":19,"values":{"1":{"slot":1,"type":"STRING","value":"source_project"},"2":{"slot":2,"type":"STRING","value":"${source_project}"}}},"20":{"slot":20,"values":{"1":{"slot":1,"type":"STRING","value":"source_dataset"},"2":{"slot":2,"type":"STRING","value":"${source_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"advanced_connection_options"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"parameter"},"2":{"slot":2,"type":"GRID","value":"value"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"ConnectionOptions"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Google BigQuery Incremental Load"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Incremental"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"39425":{"id":39425,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-231597779,"x":80,"y":-80,"width":32,"height":32,"inputConnectorIDs":[36017],"outputSuccessConnectorIDs":[36020],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"gridExportMappings":{"1":{"slot":1,"fromGrid":"required_columns","toGrid":"source_list","columnMapToFrom":{"column":"column_name","key_column":"key_column","incremental_column":"incremental_column","table":"table"},"validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Check Required Columns"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"target_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"actual_columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"source_table_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"column_name"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"required_columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"source_list"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"table"},"2":{"slot":2,"type":"GRID","value":"column"},"3":{"slot":3,"type":"GRID","value":"incremental_column"},"4":{"slot":4,"type":"GRID","value":"key_column"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"missing_columns","elements":{},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"error_message","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"39426":{"id":39426,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1096987946,"x":-80,"y":-80,"width":32,"height":32,"inputConnectorIDs":[39429],"outputSuccessConnectorIDs":[36017],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"gridExportMappings":{"1":{"slot":1,"fromGrid":"source_table_metadata","toGrid":"source_table_metadata","columnMapToFrom":{"is_nullable":"is_nullable","is_key":"is_key","length":"length","column_name":"column_name","data_type_name":"data_type_name","numeric_scale":"numeric_scale","ordinal":"ordinal"},"validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Check Source Metadata"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"metadata_stage_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${metadata_stage_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"target_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"get_metadata_sql","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${get_metadata_sql}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"stage_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_database}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"stage_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"stage_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_project}"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"stage_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"source_table_metadata","elements":{},"visible":true,"status":{},"columnNames":[]},"10":{"slot":10,"name":"error_message","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"39814":{"id":39814,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":947479920,"x":240,"y":-80,"width":32,"height":32,"inputConnectorIDs":[36020],"outputSuccessConnectorIDs":[39910],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"SELECT_STRING","mapTo":"source_column_list","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"incremental_column","mapTo":"incremental_column","validationStatus":null,"validationMessage":null},"3":{"slot":3,"fromId":null,"fromName":"incremental_column_count","mapTo":"incremental_column_count","validationStatus":null,"validationMessage":null},"4":{"slot":4,"fromId":null,"fromName":"key_column_count","mapTo":"key_column_count","validationStatus":null,"validationMessage":null}},"gridExportMappings":{"1":{"slot":1,"fromGrid":"key_columns","toGrid":"key_columns","columnMapToFrom":{"columnName":"column_name"},"validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Parameter Parser"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"platform","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_platform}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"incremental_source_list","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"source_list"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"table"},"2":{"slot":2,"type":"GRID","value":"column"},"3":{"slot":3,"type":"GRID","value":"incremental_column"},"4":{"slot":4,"type":"GRID","value":"key_column"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"source_list","elements":{},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"stage_warehouse","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_warehouse}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"stage_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_database}"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"stage_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"stage_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_project}"}}}},"visible":true,"status":{},"columnNames":[]},"10":{"slot":10,"name":"stage_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"11":{"slot":11,"name":"stage_prefix","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_prefix}"}}}},"visible":true,"status":{},"columnNames":[]},"12":{"slot":12,"name":"target_warehouse","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_warehouse}"}}}},"visible":true,"status":{},"columnNames":[]},"13":{"slot":13,"name":"target_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"14":{"slot":14,"name":"target_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"15":{"slot":15,"name":"target_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_project}"}}}},"visible":true,"status":{},"columnNames":[]},"16":{"slot":16,"name":"target_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"17":{"slot":17,"name":"target_prefix","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_prefix}"}}}},"visible":true,"status":{},"columnNames":[]},"18":{"slot":18,"name":"maximum_incremental_columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"DECIMAL","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"19":{"slot":19,"name":"source_enclosing_char_e","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"`"}}}},"visible":true,"status":{},"columnNames":[]},"20":{"slot":20,"name":"source_enclosing_char_s","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"`"}}}},"visible":true,"status":{},"columnNames":[]},"21":{"slot":21,"name":"metadata_stage_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"22":{"slot":22,"name":"stage_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"23":{"slot":23,"name":"target_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"24":{"slot":24,"name":"column_count","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"DECIMAL","value":"0"}}}},"visible":true,"status":{},"columnNames":[]},"25":{"slot":25,"name":"key_column_count","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"DECIMAL","value":"0"}}}},"visible":true,"status":{},"columnNames":[]},"26":{"slot":26,"name":"key_columns","elements":{},"visible":true,"status":{},"columnNames":[]},"27":{"slot":27,"name":"incremental_column_count","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"DECIMAL","value":"0"}}}},"visible":true,"status":{},"columnNames":[]},"28":{"slot":28,"name":"incremental_column","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"29":{"slot":29,"name":"SELECT_STRING","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"30":{"slot":30,"name":"stage_table_metadata_sql","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"39909":{"id":39909,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186960,"x":400,"y":-80,"width":32,"height":32,"inputConnectorIDs":[39910],"outputSuccessConnectorIDs":[40182],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Set stage_single_sql"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"# this is the only Python script we have to do this in\nif not log_metrics:\n  context.updateVariable('log_metrics', 'TRUE')\nelse:\n  context.updateVariable('log_metrics', str(log_metrics).upper())  \n\n\n# set base stage_single_sql\n## NOTE: the ` ` for BQ Syntax \nstage_single_sql = (f'SELECT {source_column_list} \\n' + \n                    f'FROM `{table_name}` \\n' +\n                    f'LIMIT 1'\n)\n\ncontext.updateVariable('stage_single_sql', stage_single_sql)\nprint(f\"Updated Variable 'stage_single_sql' with {stage_single_sql}\")\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"39969":{"id":39969,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":560,"y":-80,"width":32,"height":32,"inputConnectorIDs":[40182],"outputSuccessConnectorIDs":[40187],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sample 1 Row"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"util - Query Source"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"ctx"},"2":{"slot":2,"type":"STRING","value":"${target_table_name}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"sql"},"2":{"slot":2,"type":"STRING","value":"${stage_single_sql}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"warehouse"},"2":{"slot":2,"type":"STRING","value":"${stage_warehouse}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"database"},"2":{"slot":2,"type":"STRING","value":"${stage_database}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"schema"},"2":{"slot":2,"type":"STRING","value":"${stage_schema}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"project"},"2":{"slot":2,"type":"STRING","value":"${stage_project}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"dataset"},"2":{"slot":2,"type":"STRING","value":"${stage_dataset}"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"table"},"2":{"slot":2,"type":"STRING","value":"${stage_table_name}"}}},"9":{"slot":9,"values":{"1":{"slot":1,"type":"STRING","value":"bucket_name"},"2":{"slot":2,"type":"STRING","value":"${bucket_name}"}}},"10":{"slot":10,"values":{"1":{"slot":1,"type":"STRING","value":"dist_style"},"2":{"slot":2,"type":"STRING","value":"${dist_style}"}}},"11":{"slot":11,"values":{"1":{"slot":1,"type":"STRING","value":"encryption_method"},"2":{"slot":2,"type":"STRING","value":"${encryption_method}"}}},"12":{"slot":12,"values":{"1":{"slot":1,"type":"STRING","value":"kms_key_id"},"2":{"slot":2,"type":"STRING","value":"${kms_key_id}"}}},"13":{"slot":13,"values":{"1":{"slot":1,"type":"STRING","value":"staging_type"},"2":{"slot":2,"type":"STRING","value":"${staging_type}"}}},"14":{"slot":14,"values":{"1":{"slot":1,"type":"STRING","value":"storage_account"},"2":{"slot":2,"type":"STRING","value":"${storage_account}"}}},"15":{"slot":15,"values":{"1":{"slot":1,"type":"STRING","value":"blob_container"},"2":{"slot":2,"type":"STRING","value":"${blob_container}"}}},"16":{"slot":16,"values":{"1":{"slot":1,"type":"STRING","value":"cloud_storage_area"},"2":{"slot":2,"type":"STRING","value":"${cloud_storage_area}"}}},"17":{"slot":17,"values":{"1":{"slot":1,"type":"STRING","value":"oauth"},"2":{"slot":2,"type":"STRING","value":"${oauth}"}}},"18":{"slot":18,"values":{"1":{"slot":1,"type":"STRING","value":"z_bq_dialect"},"2":{"slot":2,"type":"STRING","value":"Standard"}}},"19":{"slot":19,"values":{"1":{"slot":1,"type":"STRING","value":"source_project"},"2":{"slot":2,"type":"STRING","value":"${source_project}"}}},"20":{"slot":20,"values":{"1":{"slot":1,"type":"STRING","value":"source_dataset"},"2":{"slot":2,"type":"STRING","value":"${source_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"advanced_connection_options"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"parameter"},"2":{"slot":2,"type":"GRID","value":"value"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"ConnectionOptions"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Google BigQuery Incremental Load"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Incremental"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"40186":{"id":40186,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":332733136,"x":720,"y":-80,"width":32,"height":32,"inputConnectorIDs":[40187],"outputSuccessConnectorIDs":[40294],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"load_strategy","mapTo":"load_strategy","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"stage_sql","mapTo":"stage_sql","validationStatus":null,"validationMessage":null},"3":{"slot":3,"fromId":null,"fromName":"inc_col_type","mapTo":"inc_col_type","validationStatus":null,"validationMessage":null},"4":{"slot":4,"fromId":null,"fromName":"max_value","mapTo":"max_value","validationStatus":null,"validationMessage":null}},"gridExportMappings":{"1":{"slot":1,"fromGrid":"stage_table_metadata","toGrid":"stage_table_metadata","columnMapToFrom":{"type":"type","column_name":"column_name"},"validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Decide Load Strategy"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"source_column_list","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${source_column_list}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"key_column_count","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${key_column_count}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"incremental_column_count","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${incremental_column_count}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"incremental_column","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${incremental_column}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"skip_inc_col_check","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"N"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"timestamp_precision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"microseconds"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"safely_cast_datetime_to_date","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Y"}}}},"visible":true,"status":{},"columnNames":[]},"10":{"slot":10,"name":"encloseTableString","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"TRUE"}}}},"visible":true,"status":{},"columnNames":[]},"11":{"slot":11,"name":"source_enclosing_char_s","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"`"}}}},"visible":true,"status":{},"columnNames":[]},"12":{"slot":12,"name":"source_enclosing_char_e","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"`"}}}},"visible":true,"status":{},"columnNames":[]},"13":{"slot":13,"name":"target_platform","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_platform}"}}}},"visible":true,"status":{},"columnNames":[]},"14":{"slot":14,"name":"stage_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_database}"}}}},"visible":true,"status":{},"columnNames":[]},"15":{"slot":15,"name":"stage_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"16":{"slot":16,"name":"stage_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_project}"}}}},"visible":true,"status":{},"columnNames":[]},"17":{"slot":17,"name":"stage_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"18":{"slot":18,"name":"stage_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"19":{"slot":19,"name":"target_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"20":{"slot":20,"name":"target_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"21":{"slot":21,"name":"target_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_project}"}}}},"visible":true,"status":{},"columnNames":[]},"22":{"slot":22,"name":"target_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"23":{"slot":23,"name":"target_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"24":{"slot":24,"name":"fix_target_data_types","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${fix_target_data_types}"}}}},"visible":true,"status":{},"columnNames":[]},"25":{"slot":25,"name":"load_strategy","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"26":{"slot":26,"name":"max_value","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"27":{"slot":27,"name":"stage_sql","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"28":{"slot":28,"name":"inc_col_type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"29":{"slot":29,"name":"stage_table_metadata","elements":{},"visible":true,"status":{},"columnNames":[]},"30":{"slot":30,"name":"messages","elements":{},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"40282":{"id":40282,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":880,"y":-80,"width":32,"height":32,"inputConnectorIDs":[40294],"outputSuccessConnectorIDs":[40522],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"queryRowCount","mapTo":"static_row_count","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Stage Data"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"util - Query Source"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"ctx"},"2":{"slot":2,"type":"STRING","value":"${target_table_name}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"sql"},"2":{"slot":2,"type":"STRING","value":"${stage_sql}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"warehouse"},"2":{"slot":2,"type":"STRING","value":"${stage_warehouse}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"database"},"2":{"slot":2,"type":"STRING","value":"${stage_database}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"schema"},"2":{"slot":2,"type":"STRING","value":"${stage_schema}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"project"},"2":{"slot":2,"type":"STRING","value":"${stage_project}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"dataset"},"2":{"slot":2,"type":"STRING","value":"${stage_dataset}"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"table"},"2":{"slot":2,"type":"STRING","value":"${stage_table_name}"}}},"9":{"slot":9,"values":{"1":{"slot":1,"type":"STRING","value":"bucket_name"},"2":{"slot":2,"type":"STRING","value":"${bucket_name}"}}},"10":{"slot":10,"values":{"1":{"slot":1,"type":"STRING","value":"dist_style"},"2":{"slot":2,"type":"STRING","value":"${dist_style}"}}},"11":{"slot":11,"values":{"1":{"slot":1,"type":"STRING","value":"encryption_method"},"2":{"slot":2,"type":"STRING","value":"${encryption_method}"}}},"12":{"slot":12,"values":{"1":{"slot":1,"type":"STRING","value":"kms_key_id"},"2":{"slot":2,"type":"STRING","value":"${kms_key_id}"}}},"13":{"slot":13,"values":{"1":{"slot":1,"type":"STRING","value":"staging_type"},"2":{"slot":2,"type":"STRING","value":"${staging_type}"}}},"14":{"slot":14,"values":{"1":{"slot":1,"type":"STRING","value":"storage_account"},"2":{"slot":2,"type":"STRING","value":"${storage_account}"}}},"15":{"slot":15,"values":{"1":{"slot":1,"type":"STRING","value":"blob_container"},"2":{"slot":2,"type":"STRING","value":"${blob_container}"}}},"16":{"slot":16,"values":{"1":{"slot":1,"type":"STRING","value":"cloud_storage_area"},"2":{"slot":2,"type":"STRING","value":"${cloud_storage_area}"}}},"17":{"slot":17,"values":{"1":{"slot":1,"type":"STRING","value":"oauth"},"2":{"slot":2,"type":"STRING","value":"${oauth}"}}},"18":{"slot":18,"values":{"1":{"slot":1,"type":"STRING","value":"z_bq_dialect"},"2":{"slot":2,"type":"STRING","value":"Standard"}}},"19":{"slot":19,"values":{"1":{"slot":1,"type":"STRING","value":"source_project"},"2":{"slot":2,"type":"STRING","value":"${source_project}"}}},"20":{"slot":20,"values":{"1":{"slot":1,"type":"STRING","value":"source_dataset"},"2":{"slot":2,"type":"STRING","value":"${source_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"advanced_connection_options"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"parameter"},"2":{"slot":2,"type":"GRID","value":"value"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"ConnectionOptions"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Google BigQuery Incremental Load"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Incremental"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"40473":{"id":40473,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-313802845,"x":1280,"y":-160,"width":32,"height":32,"inputConnectorIDs":[40548],"outputSuccessConnectorIDs":[40611],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Load Target"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"platform","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_platform}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"stage_warehouse","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_warehouse}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"stage_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_database}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"stage_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"stage_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_project}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"stage_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"stage_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"target_warehouse","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_warehouse}"}}}},"visible":true,"status":{},"columnNames":[]},"10":{"slot":10,"name":"target_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"11":{"slot":11,"name":"target_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"12":{"slot":12,"name":"target_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_project}"}}}},"visible":true,"status":{},"columnNames":[]},"13":{"slot":13,"name":"target_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"14":{"slot":14,"name":"target_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"15":{"slot":15,"name":"load_strategy","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${load_strategy}"}}}},"visible":true,"status":{},"columnNames":[]},"16":{"slot":16,"name":"dist_style","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${dist_style}"}}}},"visible":true,"status":{},"columnNames":[]},"17":{"slot":17,"name":"columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"stage_table_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"column_name"}}}},"visible":true,"status":{},"columnNames":[]},"18":{"slot":18,"name":"key_columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"key_columns"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"columnName"}}}},"visible":true,"status":{},"columnNames":[]},"19":{"slot":19,"name":"incremental_column","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"20":{"slot":20,"name":"join_statement","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"21":{"slot":21,"name":"log_metrics","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${log_metrics}"}}}},"visible":true,"status":{},"columnNames":[]},"22":{"slot":22,"name":"static_row_count","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${static_row_count}"}}}},"visible":true,"status":{},"columnNames":[]},"23":{"slot":23,"name":"logged_rows_inserted","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"DECIMAL","value":"0"}}}},"visible":true,"status":{},"columnNames":[]},"24":{"slot":24,"name":"logged_rows_update","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"DECIMAL","value":"0"}}}},"visible":true,"status":{},"columnNames":[]},"25":{"slot":25,"name":"logged_rows_deleted","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"DECIMAL","value":"0"}}}},"visible":true,"status":{},"columnNames":[]},"26":{"slot":26,"name":"sub_RangeStartDate","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"40521":{"id":40521,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":1040,"y":-80,"width":32,"height":32,"inputConnectorIDs":[40522],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[40548],"outputFalseConnectorIDs":[40621],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Data Staged?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"static_row_count"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Greater than"},"4":{"slot":4,"type":"STRING","value":"0"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"40573":{"id":40573,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-2112089365,"x":1424,"y":0,"width":32,"height":32,"inputConnectorIDs":[40838],"outputSuccessConnectorIDs":[40606],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log 0"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Metrics","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"inserted"},"2":{"slot":2,"type":"STRING","value":"Numeric"},"3":{"slot":3,"type":"STRING","value":"0"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"deleted"},"2":{"slot":2,"type":"STRING","value":"Numeric"},"3":{"slot":3,"type":"STRING","value":"0"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"updated"},"2":{"slot":2,"type":"STRING","value":"Numeric"},"3":{"slot":3,"type":"STRING","value":"0"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"40585":{"id":40585,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186960,"x":1120,"y":0,"width":32,"height":32,"inputConnectorIDs":[40621],"outputSuccessConnectorIDs":[40793],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Warn 0 Rows"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"print(f\"No rows have been staged, skipping attempt to load data into '{target_table_name}'\")"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"40605":{"id":40605,"inputCardinality":"MANY","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1343684451,"x":1520,"y":-80,"width":32,"height":32,"inputConnectorIDs":[40611,40606],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[40639],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"_"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"40638":{"id":40638,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":1680,"y":-80,"width":32,"height":32,"inputConnectorIDs":[40639],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"40792":{"id":40792,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":1280,"y":0,"width":32,"height":32,"inputConnectorIDs":[40793],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[40838],"outputFalseConnectorIDs":[40814],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Metrics?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"log_metrics"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"TRUE"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"40810":{"id":40810,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":1280,"y":80,"width":32,"height":32,"inputConnectorIDs":[40814],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End: No Rows Loaded"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"36013":{"id":36013,"sourceID":39400,"targetID":39424},"36017":{"id":36017,"sourceID":39426,"targetID":39425},"36020":{"id":36020,"sourceID":39425,"targetID":39814},"39429":{"id":39429,"sourceID":39424,"targetID":39426},"39910":{"id":39910,"sourceID":39814,"targetID":39909},"40182":{"id":40182,"sourceID":39909,"targetID":39969},"40187":{"id":40187,"sourceID":39969,"targetID":40186},"40294":{"id":40294,"sourceID":40186,"targetID":40282},"40522":{"id":40522,"sourceID":40282,"targetID":40521},"40606":{"id":40606,"sourceID":40573,"targetID":40605},"40611":{"id":40611,"sourceID":40473,"targetID":40605},"40793":{"id":40793,"sourceID":40585,"targetID":40792}},"failureConnectors":{},"unconditionalConnectors":{"36009":{"id":36009,"sourceID":39399,"targetID":39400},"40639":{"id":40639,"sourceID":40605,"targetID":40638}},"trueConnectors":{"40548":{"id":40548,"sourceID":40521,"targetID":40473},"40838":{"id":40838,"sourceID":40792,"targetID":40573}},"falseConnectors":{"40621":{"id":40621,"sourceID":40521,"targetID":40585},"40814":{"id":40814,"sourceID":40792,"targetID":40810}},"iterationConnectors":{},"noteConnectors":{},"canUndo":true,"undoCommand":"Move Components","undoCreated":1607429049263,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"40560":{"id":40560,"x":1840,"y":-391,"width":250,"height":58,"text":"**IGNORE**","colour":"d60000"},"40562":{"id":40562,"x":-878,"y":94,"width":250,"height":58,"text":"**IGNORE**","colour":"d60000"}},"variables":{"oauth":{"definition":{"name":"oauth","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"max_value":{"definition":{"name":"max_value","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"stage_sql":{"definition":{"name":"stage_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"dist_style":{"definition":{"name":"dist_style","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"kms_key_id":{"definition":{"name":"kms_key_id","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"table_name":{"definition":{"name":"table_name","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"bucket_name":{"definition":{"name":"bucket_name","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"log_metrics":{"definition":{"name":"log_metrics","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"TRUE"},"inc_col_type":{"definition":{"name":"inc_col_type","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"stage_prefix":{"definition":{"name":"stage_prefix","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_schema":{"definition":{"name":"stage_schema","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"staging_type":{"definition":{"name":"staging_type","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"Snowflake Managed"},"load_strategy":{"definition":{"name":"load_strategy","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"stage_dataset":{"definition":{"name":"stage_dataset","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_project":{"definition":{"name":"stage_project","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_prefix":{"definition":{"name":"target_prefix","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_schema":{"definition":{"name":"target_schema","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"blob_container":{"definition":{"name":"blob_container","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"source_dataset":{"definition":{"name":"source_dataset","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"source_project":{"definition":{"name":"source_project","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_database":{"definition":{"name":"stage_database","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_dataset":{"definition":{"name":"target_dataset","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_project":{"definition":{"name":"target_project","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_warehouse":{"definition":{"name":"stage_warehouse","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"storage_account":{"definition":{"name":"storage_account","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"target_database":{"definition":{"name":"target_database","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_platform":{"definition":{"name":"target_platform","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"Redshift"},"get_metadata_sql":{"definition":{"name":"get_metadata_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"key_column_count":{"definition":{"name":"key_column_count","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"0"},"stage_single_sql":{"definition":{"name":"stage_single_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"stage_table_name":{"definition":{"name":"stage_table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"static_row_count":{"definition":{"name":"static_row_count","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"0"},"target_warehouse":{"definition":{"name":"target_warehouse","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"encryption_method":{"definition":{"name":"encryption_method","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"None"},"target_table_name":{"definition":{"name":"target_table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"cloud_storage_area":{"definition":{"name":"cloud_storage_area","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"incremental_column":{"definition":{"name":"incremental_column","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"source_column_list":{"definition":{"name":"source_column_list","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"source_metadata_sql":{"definition":{"name":"source_metadata_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"fix_target_data_types":{"definition":{"name":"fix_target_data_types","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"incremental_column_count":{"definition":{"name":"incremental_column_count","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"0"},"metadata_stage_table_name":{"definition":{"name":"metadata_stage_table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""}},"grids":{"key_columns":{"definition":{"name":"key_columns","scope":"TASKBATCH","definitions":[{"name":"columnName","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[]},"source_list":{"definition":{"name":"source_list","scope":"TASKBATCH","definitions":[{"name":"table","type":"TEXT"},{"name":"column","type":"TEXT"},{"name":"incremental_column","type":"DECIMAL"},{"name":"key_column","type":"DECIMAL"}],"description":"","visibility":"PUBLIC"},"values":[]},"stage_table_metadata":{"definition":{"name":"stage_table_metadata","scope":"TASKBATCH","definitions":[{"name":"column_name","type":"TEXT"},{"name":"type","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[]},"source_table_metadata":{"definition":{"name":"source_table_metadata","scope":"TASKBATCH","definitions":[{"name":"column_name","type":"TEXT"},{"name":"data_type_name","type":"TEXT"},{"name":"length","type":"TEXT"},{"name":"numeric_scale","type":"TEXT"},{"name":"is_key","type":"TEXT"},{"name":"is_nullable","type":"TEXT"},{"name":"ordinal","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[]},"advanced_connection_options":{"definition":{"name":"advanced_connection_options","scope":"BRANCH","definitions":[{"name":"parameter","type":"TEXT"},{"name":"value","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]}}},"Google BigQuery - 2 - Stage Object":{"id":-2,"revision":8,"created":1607429042483,"timestamp":1607429042483,"components":{"39659":{"id":39659,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":-2960,"y":0,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[41622],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"41602":{"id":41602,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":-2799,"y":-4,"width":32,"height":32,"inputConnectorIDs":[41622],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[42469],"outputUnconditionalConnectorIDs":[42462],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"stage_table_name","mapTo":"stage_table_name","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"metadata_stage_table_name","mapTo":"metadata_stage_table_name","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Run Job"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Google BigQuery - 3 - Stage Object"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"bucket_name"},"2":{"slot":2,"type":"STRING","value":"${bucket_name}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"stage_prefix"},"2":{"slot":2,"type":"STRING","value":"${stage_prefix}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"stage_schema"},"2":{"slot":2,"type":"STRING","value":"${stage_schema}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"target_prefix"},"2":{"slot":2,"type":"STRING","value":"${target_prefix}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"target_schema"},"2":{"slot":2,"type":"STRING","value":"${target_schema}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"table_name"},"2":{"slot":2,"type":"STRING","value":"${table_name}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"dist_style"},"2":{"slot":2,"type":"STRING","value":"${dist_style}"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"target_platform"},"2":{"slot":2,"type":"STRING","value":"${target_platform}"}}},"9":{"slot":9,"values":{"1":{"slot":1,"type":"STRING","value":"target_database"},"2":{"slot":2,"type":"STRING","value":"${target_database}"}}},"10":{"slot":10,"values":{"1":{"slot":1,"type":"STRING","value":"target_warehouse"},"2":{"slot":2,"type":"STRING","value":"${target_warehouse}"}}},"11":{"slot":11,"values":{"1":{"slot":1,"type":"STRING","value":"stage_database"},"2":{"slot":2,"type":"STRING","value":"${stage_database}"}}},"12":{"slot":12,"values":{"1":{"slot":1,"type":"STRING","value":"stage_warehouse"},"2":{"slot":2,"type":"STRING","value":"${stage_warehouse}"}}},"13":{"slot":13,"values":{"1":{"slot":1,"type":"STRING","value":"target_dataset"},"2":{"slot":2,"type":"STRING","value":"${target_dataset}"}}},"14":{"slot":14,"values":{"1":{"slot":1,"type":"STRING","value":"target_project"},"2":{"slot":2,"type":"STRING","value":"${target_project}"}}},"15":{"slot":15,"values":{"1":{"slot":1,"type":"STRING","value":"stage_dataset"},"2":{"slot":2,"type":"STRING","value":"${stage_dataset}"}}},"16":{"slot":16,"values":{"1":{"slot":1,"type":"STRING","value":"stage_project"},"2":{"slot":2,"type":"STRING","value":"${stage_project}"}}},"17":{"slot":17,"values":{"1":{"slot":1,"type":"STRING","value":"cloud_storage_area"},"2":{"slot":2,"type":"STRING","value":"${cloud_storage_area}"}}},"18":{"slot":18,"values":{"1":{"slot":1,"type":"STRING","value":"storage_account"},"2":{"slot":2,"type":"STRING","value":"${storage_account}"}}},"19":{"slot":19,"values":{"1":{"slot":1,"type":"STRING","value":"blob_container"},"2":{"slot":2,"type":"STRING","value":"${blob_container}"}}},"20":{"slot":20,"values":{"1":{"slot":1,"type":"STRING","value":"encryption_method"},"2":{"slot":2,"type":"STRING","value":"${encryption_method}"}}},"21":{"slot":21,"values":{"1":{"slot":1,"type":"STRING","value":"kms_key_id"},"2":{"slot":2,"type":"STRING","value":"${kms_key_id}"}}},"22":{"slot":22,"values":{"1":{"slot":1,"type":"STRING","value":"staging_type"},"2":{"slot":2,"type":"STRING","value":"${staging_type}"}}},"23":{"slot":23,"values":{"1":{"slot":1,"type":"STRING","value":"log_metrics"},"2":{"slot":2,"type":"STRING","value":"${log_metrics}"}}},"24":{"slot":24,"values":{"1":{"slot":1,"type":"STRING","value":"fix_target_data_types"},"2":{"slot":2,"type":"STRING","value":"${fix_target_data_types}"}}},"25":{"slot":25,"values":{"1":{"slot":1,"type":"STRING","value":"oauth"},"2":{"slot":2,"type":"STRING","value":"${oauth}"}}},"26":{"slot":26,"values":{"1":{"slot":1,"type":"STRING","value":"source_dataset"},"2":{"slot":2,"type":"STRING","value":"${source_dataset}"}}},"27":{"slot":27,"values":{"1":{"slot":1,"type":"STRING","value":"source_project"},"2":{"slot":2,"type":"STRING","value":"${source_project}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"advanced_connection_options"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"parameter"},"2":{"slot":2,"type":"GRID","value":"value"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"advanced_connection_options"},"2":{"slot":2,"type":"STRING","value":"grid"}}},"2":{"slot":2,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"source_list"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"table"},"2":{"slot":2,"type":"GRID","value":"column"},"3":{"slot":3,"type":"GRID","value":"incremental_column"},"4":{"slot":4,"type":"GRID","value":"key_column"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"source_list"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Google BigQuery Incremental Load"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Incremental"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"42104":{"id":42104,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":-2480,"y":0,"width":32,"height":32,"inputConnectorIDs":[42105],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"42142":{"id":42142,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1291941950,"x":-2640,"y":0,"width":32,"height":32,"inputConnectorIDs":[42462],"outputSuccessConnectorIDs":[42105],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Drop Staging Tables"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Tables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${metadata_stage_table_name}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Cascade","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Yes"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Ignore Missing","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Yes"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"42464":{"id":42464,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":-2800,"y":80,"width":32,"height":32,"inputConnectorIDs":[42469],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End: Failure"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"42105":{"id":42105,"sourceID":42142,"targetID":42104}},"failureConnectors":{"42469":{"id":42469,"sourceID":41602,"targetID":42464}},"unconditionalConnectors":{"41622":{"id":41622,"sourceID":39659,"targetID":41602},"42462":{"id":42462,"sourceID":41602,"targetID":42142}},"trueConnectors":{},"falseConnectors":{},"iterationConnectors":{},"noteConnectors":{},"canUndo":true,"undoCommand":"Move Components","undoCreated":1607429041384,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"41612":{"id":41612,"x":-3361,"y":258,"width":250,"height":58,"text":"**IGNORE**","colour":"d60000"},"41617":{"id":41617,"x":-2023,"y":-326,"width":250,"height":58,"text":"**IGNORE**","colour":"d60000"}},"variables":{"oauth":{"definition":{"name":"oauth","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"dist_style":{"definition":{"name":"dist_style","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"kms_key_id":{"definition":{"name":"kms_key_id","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"table_name":{"definition":{"name":"table_name","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"bucket_name":{"definition":{"name":"bucket_name","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"log_metrics":{"definition":{"name":"log_metrics","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_prefix":{"definition":{"name":"stage_prefix","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_schema":{"definition":{"name":"stage_schema","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"staging_type":{"definition":{"name":"staging_type","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_dataset":{"definition":{"name":"stage_dataset","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_project":{"definition":{"name":"stage_project","type":"TEXT","scope":"BRANCH","description":"   ","visibility":"PUBLIC"},"value":""},"target_prefix":{"definition":{"name":"target_prefix","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_schema":{"definition":{"name":"target_schema","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"blob_container":{"definition":{"name":"blob_container","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"connection_url":{"definition":{"name":"connection_url","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"source_dataset":{"definition":{"name":"source_dataset","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"source_project":{"definition":{"name":"source_project","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_database":{"definition":{"name":"stage_database","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_dataset":{"definition":{"name":"target_dataset","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_project":{"definition":{"name":"target_project","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_warehouse":{"definition":{"name":"stage_warehouse","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"storage_account":{"definition":{"name":"storage_account","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_database":{"definition":{"name":"target_database","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_platform":{"definition":{"name":"target_platform","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"Redshift"},"stage_table_name":{"definition":{"name":"stage_table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"target_warehouse":{"definition":{"name":"target_warehouse","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"encryption_method":{"definition":{"name":"encryption_method","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"cloud_storage_area":{"definition":{"name":"cloud_storage_area","type":"TEXT","scope":"BRANCH","description":"The cloud storage area name that should be used when loading bq tables.","visibility":"PUBLIC"},"value":""},"metadata_stage_table_name":{"definition":{"name":"metadata_stage_table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""}},"grids":{"source_list":{"definition":{"name":"source_list","scope":"TASKBATCH","definitions":[{"name":"table","type":"TEXT"},{"name":"column","type":"TEXT"},{"name":"incremental_column","type":"DECIMAL"},{"name":"key_column","type":"DECIMAL"}],"description":"","visibility":"PUBLIC"},"values":[]},"advanced_connection_options":{"definition":{"name":"advanced_connection_options","scope":"TASKBATCH","definitions":[{"name":"parameter","type":"TEXT"},{"name":"value","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]}}}},"transformationJobs":{}}],"version":"feature_EMD_15915","environment":"redshift"}