{"objects":[{"metadata":{"identifier":{"packageName":"Matillion.Incremental","name":"HubSpot Incremental Load","revision":1,"type":"DYNAMIC"},"rootJobReference":{"name":"HubSpot - 1 - Orchestrator","type":"ORCHESTRATION","parameterMetadata":[{"slot":2,"variableName":"oauth","variableType":"SCALAR","displayName":"OAuth","description":"The name of the OAuth entry to be used from the Matillion OAuth Manager to connect to the source. These must be set up in advance, using Project --> Mangage OAuth.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":3,"variableName":"source_list","variableType":"GRID","displayName":"Tables and Columns","description":"Contains the list of tables and columns (and in some cases an incremental_column to specify which column the load should be incremented on [accepts values of 0 or 1]) to be processed. ","defaultValue":[{"values":{"1":""}}],"defaultValueType":null,"required":true},{"slot":4,"variableName":"advanced_connection_options","variableType":"GRID","displayName":"Connection Options","description":"A list of values and parameters. Parameters and their allowed values are database/driver specific. Referring to the data model will provide insight of what you could provide here.\n<br>\nThey are usually not required as sensible defaults are assumed.","defaultValue":[{"values":{"1":""}}],"defaultValueType":null,"required":false},{"slot":5,"variableName":"load_concurrent","variableType":"SCALAR","displayName":"Load Type","description":"Sequential - Iterations are done in sequence, waiting for each to complete before starting the next. \nThis is the default.\n<br>\nConcurrent - Iterations are run concurrently. This requires all \"Variables to Iterate\" to be defined as\n<a href=\"https://redshiftsupport.matillion.com/customer/portal/articles/2037630?b_id=8915#header1\" target=\"_blank\"><i>Copied</i> variables</a>, so that each iteration gets its own copy of the variable isolated from the same\nvariable being used by other concurrent executions.\n<span class=\"env rs sf\"><br><b>Note:</b> The maximum concurrency is limited by the number of available threads (2x the number of virtual cpus on your cloud instance).</span>","defaultValue":[{"values":{"1":"Sequential"}}],"defaultValueType":"TEXT","required":true},{"slot":6,"variableName":"stage_warehouse","variableType":"SCALAR","displayName":"Stage Warehouse","description":"The warehouse name where the staging data will be stored.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":7,"variableName":"stage_database","variableType":"SCALAR","displayName":"Stage Database","description":"The database name where the staging data will be stored.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":8,"variableName":"stage_schema","variableType":"SCALAR","displayName":"Stage Schema","description":"The schema name where the staging data will be stored.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":9,"variableName":"stage_prefix","variableType":"SCALAR","displayName":"Stage Prefix","description":"A prefix value that will be added to the start of the stage table names.\n<br>\ne.g. If a Stage Prefix of 'stage_' is specified and the table being processed is named 'test_data' then the target table will be named 'stage_test_data'.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":10,"variableName":"target_warehouse","variableType":"SCALAR","displayName":"Target Warehouse","description":"The warehouse name where the target data will be stored.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":11,"variableName":"target_database","variableType":"SCALAR","displayName":"Target Database","description":"The database name where the target data will be stored.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":12,"variableName":"target_schema","variableType":"SCALAR","displayName":"Target Schema","description":"The schema name where the target data will be stored.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":13,"variableName":"target_prefix","variableType":"SCALAR","displayName":"Target Prefix","description":"A prefix value that will be added to the start of the target table names.\n<br>\ne.g. If a Target Prefix of 'target_' is specified and the table being processed is named 'test_data' then the target table will be named 'target_test_data'.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":false},{"slot":14,"variableName":"staging_type","variableType":"SCALAR","displayName":"Staging Type","description":"<b>Snowflake Managed:</b> Allow Matillion ETL to create and use a temporary internal stage on Snowflake for staging the data. This stage, along with the staged data, will cease to exist after loading is complete.<br>\n                <b>Existing Amazon S3 Location:</b> Selecting this will avail the user of properties to specify a custom staging area on S3.\n                <b>Existing Azure Blob Storage Location:</b> Selecting this will avail the user of properties to specify a custom staging area on Azure.","defaultValue":[{"values":{"1":"Snowflake Managed"}}],"defaultValueType":"TEXT","required":true},{"slot":15,"variableName":"encryption_method","variableType":"SCALAR","displayName":"Encryption","description":"","defaultValue":[{"values":{"1":"None"}}],"defaultValueType":"TEXT","required":false},{"slot":16,"variableName":"s3_bucket_name","variableType":"SCALAR","displayName":"S3 Bucket Name","description":"The name of the AWS S3 bucket that will be used for staging the source data.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":false},{"slot":17,"variableName":"kms_key_id","variableType":"SCALAR","displayName":"KMS Key ID","description":"","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":false},{"slot":18,"variableName":"storage_account","variableType":"SCALAR","displayName":"Storage Account","description":"(Azure Only) Azure Storage Account to allow Matillion ETL to specify a blob container object on Snowflake for staging data. ","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":false},{"slot":19,"variableName":"blob_container","variableType":"SCALAR","displayName":"Blob Container","description":"(Azure Only) Azure Blob Container to allow Matillion ETL to use the blob storage container object on Snowflake for staging data. ","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":false}]},"orchestrationJobs":["HubSpot - 1 - Orchestrator","HubSpot - 2 - Wrapper","HubSpot - 3 - Stage and Load","HubSpot - 4 - Query Source"],"transformationJobs":[],"imageMetadata":{"componentIcon":"iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAADMUlEQVR42t1XXUgUURjdfqnIyCio3sR8KIrCh+ilngrqwZdi2b2za4nVYvRgFo1z70w1RokFERL0UhCRYEUgZBYRFm710INP/aEUYVKCZBaiGe76dWYd1zuza+6uk0UDHzPsne+ec88933dnfb5/8SK/f87fAdaDARJKFwkWR0TJUIpymkdT8sksW5BdkggUAzSGICk6slEDoEtJVx7auUPEmZoFgeBpF7gdyqaM5+BKfUq+rmzOMJmpaQkYyvbMF8GephIIVmSWfNy/Egm9aUgMWN744wqMmSe8BmBXocYjJJ9BfEhOxNlFikTmTekBzp7YOfGsPJB+Qv8yeOC+tKIoifCq31cSK7Hf/e5NaZrmbJAw7dKEpKyHeHDrjBGQynQXJu2zJx9B1VTNKIHE5EZpAdRol7bkFrZp8cQYq8W+v0p6QGctMGGEqvwLvSOBDpcw6gSJN1CjDvef6ftIopd0gcg2j7eEHUD8mBw0JSyCO7wlwZmWBqgT/eAmxlrHvOIY+0Lcv8LD6mDvpMljqI6D5PPNcvQVwV46GxM759HJGdrimrjG/Y6qqnnX+JGjrq366JH8yiGnvM4GZYFzzp9pmnYe402Od0223Iv9lw+vuCy9dQG8GUGIWC1Xh+/ywzQgwvYBV1rgQRUoZY5V8dA6eVwIUQzwr7jfecwr3tfzY3SWq9Qj9o6SWp7nQVNSipwE4HyooOt6iUyiQau8h/HRmFDoBq8kqNEfmeJQy6YXtMkkXvPyVkty7Hu+dXLit3BcsG/j44MiRKe4Rhjf41ElMCYTiIoKgtSjeP6ELRpM6RFo0ye5FrWMOX3wE6EN7g+YNhC4hL2epBN2kxEohEIPEBe8A08cz2wfnt92inIyIHHS7RPRZJWpaZrzAd4PBUK5fWoLdsU+DfuS4EZg7fh/iWGN7dS51gGzjcB0jTCmuC6q6mDG9VJ57geRRbkY7rZrVb3j4PJVXV29ESB9WOVz3HdbciN6ZRK5Ot79wfpisncNwygAaCNiyG5GVrT7XM0qW8e3OAkEL0+VA6nnArgG0Q1ShdNTQA2uxodIM8A/w3QNZIaWZJIH6YPTBv+vr1+2ZGgUV7vrNwAAAABJRU5ErkJggg==","smallIcon":"iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAABfklEQVR42mNggIL/DAyMDOSC/5WRif8rI378r4rYgM2g/w0OLP+rI4uAamJxGbAPKPkfjKsjZDHkq6NC4fI1MepYDIjyAtp+Dajg5v/K8Ev/a6JU0SzQAMp9AOK7/8uSeHF7JS2N9X9VVB9YcXWUP6ohEcv+V0VWEhcmVZFhQFs/Af3d/j80lPl/ZbQW0IATQFeu/F8WLUOcITXhmkBN1/9XRN4A0v/gYQAK7MrIeBJcAtQENgQYU1VR5UD+VyD+jTUwMQM3YgVyyFdXV1tcrkxZCRariqolbEB1+Faw4pJYbhC/oqJia29F8YN1lTn/v1THdBPjhUqQAZ8qY2dVVVWFna9JV31bGfPyaWX8/4bK8knEGBADMuBhZcL/45WpP4Hsv9DEtr2yomIJbo21kdrAxFUFVPwFZMib6tjOJRV5X/8Bo/J+VeLM+w0JHJWVlXn4Au8pNKDKYWLl5eX+QLwWiO8BNdsTcvpCoAGPQWkBTYoRaEBFYWEhJwO1AACRK/lksZaMlwAAAABJRU5ErkJggg=="},"description":"","helpHtml":"<div class=\"mtln\"><img src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAADMUlEQVR42t1XXUgUURjdfqnIyCio3sR8KIrCh+ilngrqwZdi2b2za4nVYvRgFo1z70w1RokFERL0UhCRYEUgZBYRFm710INP/aEUYVKCZBaiGe76dWYd1zuza+6uk0UDHzPsne+ec88933dnfb5/8SK/f87fAdaDARJKFwkWR0TJUIpymkdT8sksW5BdkggUAzSGICk6slEDoEtJVx7auUPEmZoFgeBpF7gdyqaM5+BKfUq+rmzOMJmpaQkYyvbMF8GephIIVmSWfNy/Egm9aUgMWN744wqMmSe8BmBXocYjJJ9BfEhOxNlFikTmTekBzp7YOfGsPJB+Qv8yeOC+tKIoifCq31cSK7Hf/e5NaZrmbJAw7dKEpKyHeHDrjBGQynQXJu2zJx9B1VTNKIHE5EZpAdRol7bkFrZp8cQYq8W+v0p6QGctMGGEqvwLvSOBDpcw6gSJN1CjDvef6ftIopd0gcg2j7eEHUD8mBw0JSyCO7wlwZmWBqgT/eAmxlrHvOIY+0Lcv8LD6mDvpMljqI6D5PPNcvQVwV46GxM759HJGdrimrjG/Y6qqnnX+JGjrq366JH8yiGnvM4GZYFzzp9pmnYe402Od0223Iv9lw+vuCy9dQG8GUGIWC1Xh+/ywzQgwvYBV1rgQRUoZY5V8dA6eVwIUQzwr7jfecwr3tfzY3SWq9Qj9o6SWp7nQVNSipwE4HyooOt6iUyiQau8h/HRmFDoBq8kqNEfmeJQy6YXtMkkXvPyVkty7Hu+dXLit3BcsG/j44MiRKe4Rhjf41ElMCYTiIoKgtSjeP6ELRpM6RFo0ye5FrWMOX3wE6EN7g+YNhC4hL2epBN2kxEohEIPEBe8A08cz2wfnt92inIyIHHS7RPRZJWpaZrzAd4PBUK5fWoLdsU+DfuS4EZg7fh/iWGN7dS51gGzjcB0jTCmuC6q6mDG9VJ57geRRbkY7rZrVb3j4PJVXV29ESB9WOVz3HdbciN6ZRK5Ot79wfpisncNwygAaCNiyG5GVrT7XM0qW8e3OAkEL0+VA6nnArgG0Q1ShdNTQA2uxodIM8A/w3QNZIaWZJIH6YPTBv+vr1+2ZGgUV7vrNwAAAABJRU5ErkJggg==\"><h3>HubSpot Incremental Load</h3><p style='white-space: pre-line; word-wrap: break-word;'></p><h3>Properties</h3><table><tbody><tr><th>Property</th><th>Type</th><th>Description</th></tr><tr><td>OAuth</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>The name of the OAuth entry to be used from the Matillion OAuth Manager to connect to the source. These must be set up in advance, using Project --> Mangage OAuth.</td></tr><tr><td>Tables and Columns</td><td>Grid</td><td style='white-space: pre-line; word-wrap: break-word;'>Contains the list of tables and columns (and in some cases an incremental_column to specify which column the load should be incremented on [accepts values of 0 or 1]) to be processed. </td></tr><tr><td>Connection Options</td><td>Grid</td><td style='white-space: pre-line; word-wrap: break-word;'>A list of values and parameters. Parameters and their allowed values are database/driver specific. Referring to the data model will provide insight of what you could provide here.\n<br>\nThey are usually not required as sensible defaults are assumed.</td></tr><tr><td>Load Type</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>Sequential - Iterations are done in sequence, waiting for each to complete before starting the next. \nThis is the default.\n<br>\nConcurrent - Iterations are run concurrently. This requires all \"Variables to Iterate\" to be defined as\n<a href=\"https://redshiftsupport.matillion.com/customer/portal/articles/2037630?b_id=8915#header1\" target=\"_blank\"><i>Copied</i> variables</a>, so that each iteration gets its own copy of the variable isolated from the same\nvariable being used by other concurrent executions.\n<span class=\"env rs sf\"><br><b>Note:</b> The maximum concurrency is limited by the number of available threads (2x the number of virtual cpus on your cloud instance).</span></td></tr><tr><td>Stage Warehouse</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>The warehouse name where the staging data will be stored.</td></tr><tr><td>Stage Database</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>The database name where the staging data will be stored.</td></tr><tr><td>Stage Schema</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>The schema name where the staging data will be stored.</td></tr><tr><td>Stage Prefix</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>A prefix value that will be added to the start of the stage table names.\n<br>\ne.g. If a Stage Prefix of 'stage_' is specified and the table being processed is named 'test_data' then the target table will be named 'stage_test_data'.</td></tr><tr><td>Target Warehouse</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>The warehouse name where the target data will be stored.</td></tr><tr><td>Target Database</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>The database name where the target data will be stored.</td></tr><tr><td>Target Schema</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>The schema name where the target data will be stored.</td></tr><tr><td>Target Prefix</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>A prefix value that will be added to the start of the target table names.\n<br>\ne.g. If a Target Prefix of 'target_' is specified and the table being processed is named 'test_data' then the target table will be named 'target_test_data'.</td></tr><tr><td>Staging Type</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'><b>Snowflake Managed:</b> Allow Matillion ETL to create and use a temporary internal stage on Snowflake for staging the data. This stage, along with the staged data, will cease to exist after loading is complete.<br>\n                <b>Existing Amazon S3 Location:</b> Selecting this will avail the user of properties to specify a custom staging area on S3.\n                <b>Existing Azure Blob Storage Location:</b> Selecting this will avail the user of properties to specify a custom staging area on Azure.</td></tr><tr><td>Encryption</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'></td></tr><tr><td>S3 Bucket Name</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>The name of the AWS S3 bucket that will be used for staging the source data.</td></tr><tr><td>KMS Key ID</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'></td></tr><tr><td>Storage Account</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>(Azure Only) Azure Storage Account to allow Matillion ETL to specify a blob container object on Snowflake for staging data. </td></tr><tr><td>Blob Container</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>(Azure Only) Azure Blob Container to allow Matillion ETL to use the blob storage container object on Snowflake for staging data. </td></tr></tbody></table></div>","created":1702996875532},"orchestrationJobs":{"HubSpot - 4 - Query Source":{"id":-2,"revision":1,"created":1702911318834,"timestamp":1702911318834,"components":{"25629":{"id":25629,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":-160,"y":0,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[25639],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Start","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25630":{"id":25630,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":1847161714,"x":160,"y":0,"width":32,"height":32,"inputConnectorIDs":[25637],"outputSuccessConnectorIDs":[25636],"outputFailureConnectorIDs":[25641],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"Message","mapTo":"error_message","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"Time Taken To Load","mapTo":"time_taken_to_load","validationStatus":null,"validationMessage":null},"3":{"slot":3,"fromId":null,"fromName":"Time Taken To Stage","mapTo":"time_taken_to_stage","validationStatus":null,"validationMessage":null},"4":{"slot":4,"fromId":null,"fromName":"Row Count","mapTo":"row_count","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Query Source"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Basic/Advanced Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Advanced"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Authentication","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${oauth}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Data Source","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Data Selection","elements":{},"visible":false,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Data Source Filter","elements":{},"visible":false,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Combine Filters","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":false,"status":{},"columnNames":[]},"10":{"slot":10,"name":"SQL Query","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${sql}"}}}},"visible":true,"status":{},"columnNames":[]},"11":{"slot":11,"name":"Limit","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"100"}}}},"visible":false,"status":{},"columnNames":[]},"12":{"slot":12,"name":"Connection Options","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"connectionOptions"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"parameter"},"2":{"slot":2,"type":"GRID","value":"value"}}}},"visible":true,"status":{},"columnNames":[]},"13":{"slot":13,"name":"Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${database}"}}}},"visible":true,"status":{},"columnNames":[]},"14":{"slot":14,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${schema}"}}}},"visible":true,"status":{},"columnNames":[]},"15":{"slot":15,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${table}"}}}},"visible":true,"status":{},"columnNames":[]},"16":{"slot":16,"name":"Warehouse","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${warehouse}"}}}},"visible":true,"status":{},"columnNames":[]},"17":{"slot":17,"name":"S3 Staging Area","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${s3_bucket_name}"}}}},"visible":true,"status":{},"columnNames":[]},"18":{"slot":18,"name":"Primary Keys","elements":{},"visible":true,"status":{},"columnNames":[]},"20":{"slot":20,"name":"Data Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot"}}}},"visible":false,"status":{},"columnNames":[]},"106":{"slot":106,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"1001":{"slot":1001,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"1013":{"slot":1013,"name":"Auto Debug","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${autoDebug}"}}}},"visible":true,"status":{},"columnNames":[]},"1014":{"slot":1014,"name":"Debug Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${autoDebugVerbosity}"}}}},"visible":false,"status":{},"columnNames":[]},"1992":{"slot":1992,"name":"Warehouse","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"[Environment Default]"}}}},"visible":false,"status":{},"columnNames":[]},"1993":{"slot":1993,"name":"Use Accelerated Endpoint","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"False"}}}},"visible":false,"status":{},"columnNames":[]},"1994":{"slot":1994,"name":"Stage","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"1995":{"slot":1995,"name":"Stage Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"[Environment Default]"}}}},"visible":false,"status":{},"columnNames":[]},"1996":{"slot":1996,"name":"Stage Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"[Environment Default]"}}}},"visible":false,"status":{},"columnNames":[]},"1997":{"slot":1997,"name":"New Table Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"1998":{"slot":1998,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"[Environment Default]"}}}},"visible":false,"status":{},"columnNames":[]},"1999":{"slot":1999,"name":"Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"[Environment Default]"}}}},"visible":false,"status":{},"columnNames":[]},"2000":{"slot":2000,"name":"Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Standard"}}}},"visible":true,"status":{},"columnNames":[]},"40000":{"slot":40000,"name":"Encryption","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${encryption_method}"}}}},"visible":true,"status":{},"columnNames":[]},"40001":{"slot":40001,"name":"KMS Key ID","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${kms_key_id}"}}}},"visible":false,"status":{},"columnNames":[]},"40501":{"slot":40501,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"40502":{"slot":40502,"name":"Load Options","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"On"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"Off"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"On"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"On"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"Gzip"}}}},"visible":true,"status":{},"columnNames":[]},"63319":{"slot":63319,"name":"Stage Authentication","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Credentials"}}}},"visible":false,"status":{},"columnNames":[]},"63320":{"slot":63320,"name":"Storage Account","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${storage_account}"}}}},"visible":false,"status":{},"columnNames":[]},"63321":{"slot":63321,"name":"Blob Container","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${blob_container}"}}}},"visible":false,"status":{},"columnNames":[]},"63322":{"slot":63322,"name":"Storage Integration","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"84533":{"slot":84533,"name":"Storage Integration","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"88340":{"slot":88340,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"88341":{"slot":88341,"name":"Stage Platform","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${staging_type}"}}}},"visible":true,"status":{},"columnNames":[]},"88342":{"slot":88342,"name":"Stage Authentication","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Credentials"}}}},"visible":true,"status":{},"columnNames":[]},"88343":{"slot":88343,"name":"Storage Integration","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"88344":{"slot":88344,"name":"Use Accelerated Endpoint","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"False"}}}},"visible":false,"status":{},"columnNames":[]},"88345":{"slot":88345,"name":"Stage","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"[Custom]"}}}},"visible":true,"status":{},"columnNames":[]},"98776":{"slot":98776,"name":"GCS Staging Area","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${cloud_storage_area}"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25631":{"id":25631,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":166,"y":160,"width":32,"height":32,"inputConnectorIDs":[25641],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[25640],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Print Error"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"msg = \\\n\"\"\"There was an issue whilst retrieving data from the source. \nThe attempted query was: \\n{q}\nError: {e}\n\"\"\".format(q = sql, e = error_message) \nprint(msg)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25632":{"id":25632,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":160,"y":320,"width":32,"height":32,"inputConnectorIDs":[25640],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[25638],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Error"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"error"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Error Staging Data"},"2":{"slot":2,"type":"STRING","value":"${error_message}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25633":{"id":25633,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":320,"y":320,"width":32,"height":32,"inputConnectorIDs":[25638],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Failure"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25634":{"id":25634,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":320,"y":0,"width":32,"height":32,"inputConnectorIDs":[25636],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25635":{"id":25635,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":4,"y":3,"width":32,"height":32,"inputConnectorIDs":[25639],"outputSuccessConnectorIDs":[25637],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Limit HubSpot Data"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"# hard limit to HubSpot tables that can be set during testing (via `testlimit` connection option)\nif dataLimit:\n  sql += \" LIMIT {}\".format(dataLimit)  \n  context.updateVariable('sql', sql)\n  print(\"New SQL: {}\".format(sql))\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"25636":{"id":25636,"sourceID":25630,"targetID":25634},"25637":{"id":25637,"sourceID":25635,"targetID":25630}},"failureConnectors":{"25641":{"id":25641,"sourceID":25630,"targetID":25631}},"unconditionalConnectors":{"25638":{"id":25638,"sourceID":25632,"targetID":25633},"25639":{"id":25639,"sourceID":25629,"targetID":25635},"25640":{"id":25640,"sourceID":25631,"targetID":25632}},"trueConnectors":{},"falseConnectors":{},"iterationConnectors":{},"noteConnectors":{},"canUndo":false,"undoCommand":"","undoCreated":0,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{},"variables":{"sql":{"definition":{"name":"sql","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"oauth":{"definition":{"name":"oauth","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"table":{"definition":{"name":"table","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"schema":{"definition":{"name":"schema","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"dataset":{"definition":{"name":"dataset","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"project":{"definition":{"name":"project","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"database":{"definition":{"name":"database","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"autoDebug":{"definition":{"name":"autoDebug","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"Off"},"dataLimit":{"definition":{"name":"dataLimit","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"row_count":{"definition":{"name":"row_count","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"},"warehouse":{"definition":{"name":"warehouse","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"dist_style":{"definition":{"name":"dist_style","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"Even"},"kms_key_id":{"definition":{"name":"kms_key_id","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"staging_type":{"definition":{"name":"staging_type","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"Snowflake Managed"},"error_message":{"definition":{"name":"error_message","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"blob_container":{"definition":{"name":"blob_container","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"s3_bucket_name":{"definition":{"name":"s3_bucket_name","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"storage_account":{"definition":{"name":"storage_account","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"encryption_method":{"definition":{"name":"encryption_method","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"None"},"target_table_name":{"definition":{"name":"target_table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"autoDebugVerbosity":{"definition":{"name":"autoDebugVerbosity","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"3"},"cloud_storage_area":{"definition":{"name":"cloud_storage_area","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"time_taken_to_load":{"definition":{"name":"time_taken_to_load","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"},"time_taken_to_stage":{"definition":{"name":"time_taken_to_stage","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"}},"grids":{"connectionOptions":{"definition":{"name":"connectionOptions","scope":"BRANCH","definitions":[{"name":"parameter","type":"TEXT"},{"name":"value","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]}}},"HubSpot - 2 - Wrapper":{"id":-2,"revision":1,"created":1702911318834,"timestamp":1702911318834,"components":{"25564":{"id":25564,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":-240,"y":-80,"width":32,"height":32,"inputConnectorIDs":[25575],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[25577],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log End"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"info"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Iteration Duration"},"2":{"slot":2,"type":"STRING","value":"${iter_duration}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"Iteration End"},"2":{"slot":2,"type":"STRING","value":"${iter_state}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"Iteration Name"},"2":{"slot":2,"type":"STRING","value":"${table_name}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25566":{"id":25566,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":-400,"y":-160,"width":32,"height":32,"inputConnectorIDs":[25573],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Iteration Success"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25567":{"id":25567,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":80,"y":-80,"width":32,"height":32,"inputConnectorIDs":[25579],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Iteration Failure: Error Cleaning Stage Tables"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25568":{"id":25568,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":-624,"y":0,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[25576],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Start","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25569":{"id":25569,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":-400,"y":0,"width":32,"height":32,"inputConnectorIDs":[25576],"outputSuccessConnectorIDs":[25573],"outputFailureConnectorIDs":[25578,34431],"outputUnconditionalConnectorIDs":[25575],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"metadata_stage_table_name","mapTo":"metadata_stage_table_name","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"comp_message","mapTo":"iter_component_message","validationStatus":null,"validationMessage":null},"3":{"slot":3,"fromId":null,"fromName":"comp_name","mapTo":"iter_component_name","validationStatus":null,"validationMessage":null},"4":{"slot":4,"fromId":null,"fromName":"Status","mapTo":"iter_state","validationStatus":null,"validationMessage":null},"5":{"slot":5,"fromId":null,"fromName":"target_table_name","mapTo":"target_table_name","validationStatus":null,"validationMessage":null},"6":{"slot":6,"fromId":null,"fromName":"Duration","mapTo":"iter_duration","validationStatus":null,"validationMessage":null},"7":{"slot":7,"fromId":null,"fromName":"stage_table_name","mapTo":"stage_table_name","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Stage And Load"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot - 3 - Stage and Load"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"staging_type"},"2":{"slot":2,"type":"STRING","value":"${staging_type}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"blob_container"},"2":{"slot":2,"type":"STRING","value":"${blob_container}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"cloud_storage_area"},"2":{"slot":2,"type":"STRING","value":"${cloud_storage_area}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"dist_style"},"2":{"slot":2,"type":"STRING","value":"${dist_style}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"encryption_method"},"2":{"slot":2,"type":"STRING","value":"${encryption_method}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"kms_key_id"},"2":{"slot":2,"type":"STRING","value":"${kms_key_id}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"oauth"},"2":{"slot":2,"type":"STRING","value":"${oauth}"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"s3_bucket_name"},"2":{"slot":2,"type":"STRING","value":"${s3_bucket_name}"}}},"9":{"slot":9,"values":{"1":{"slot":1,"type":"STRING","value":"stage_database"},"2":{"slot":2,"type":"STRING","value":"${stage_database}"}}},"10":{"slot":10,"values":{"1":{"slot":1,"type":"STRING","value":"stage_dataset"},"2":{"slot":2,"type":"STRING","value":"${stage_dataset}"}}},"11":{"slot":11,"values":{"1":{"slot":1,"type":"STRING","value":"stage_prefix"},"2":{"slot":2,"type":"STRING","value":"${stage_prefix}"}}},"12":{"slot":12,"values":{"1":{"slot":1,"type":"STRING","value":"stage_project"},"2":{"slot":2,"type":"STRING","value":"${stage_project}"}}},"13":{"slot":13,"values":{"1":{"slot":1,"type":"STRING","value":"stage_warehouse"},"2":{"slot":2,"type":"STRING","value":"${stage_warehouse}"}}},"14":{"slot":14,"values":{"1":{"slot":1,"type":"STRING","value":"stage_schema"},"2":{"slot":2,"type":"STRING","value":"${stage_schema}"}}},"15":{"slot":15,"values":{"1":{"slot":1,"type":"STRING","value":"target_database"},"2":{"slot":2,"type":"STRING","value":"${target_database}"}}},"16":{"slot":16,"values":{"1":{"slot":1,"type":"STRING","value":"target_dataset"},"2":{"slot":2,"type":"STRING","value":"${target_dataset}"}}},"17":{"slot":17,"values":{"1":{"slot":1,"type":"STRING","value":"target_prefix"},"2":{"slot":2,"type":"STRING","value":"${target_prefix}"}}},"18":{"slot":18,"values":{"1":{"slot":1,"type":"STRING","value":"target_project"},"2":{"slot":2,"type":"STRING","value":"${target_project}"}}},"19":{"slot":19,"values":{"1":{"slot":1,"type":"STRING","value":"target_warehouse"},"2":{"slot":2,"type":"STRING","value":"${target_warehouse}"}}},"20":{"slot":20,"values":{"1":{"slot":1,"type":"STRING","value":"target_schema"},"2":{"slot":2,"type":"STRING","value":"${target_schema}"}}},"21":{"slot":21,"values":{"1":{"slot":1,"type":"STRING","value":"storage_account"},"2":{"slot":2,"type":"STRING","value":"${storage_account}"}}},"22":{"slot":22,"values":{"1":{"slot":1,"type":"STRING","value":"table_name"},"2":{"slot":2,"type":"STRING","value":"${table_name}"}}},"23":{"slot":23,"values":{"1":{"slot":1,"type":"STRING","value":"autoDebug"},"2":{"slot":2,"type":"STRING","value":"${autoDebug}"}}},"24":{"slot":24,"values":{"1":{"slot":1,"type":"STRING","value":"autoDebugVerbosity"},"2":{"slot":2,"type":"STRING","value":"${autoDebugVerbosity}"}}},"25":{"slot":25,"values":{"1":{"slot":1,"type":"STRING","value":"testLimit"},"2":{"slot":2,"type":"STRING","value":"${testLimit}"}}},"26":{"slot":26,"values":{"1":{"slot":1,"type":"STRING","value":"target_platform"},"2":{"slot":2,"type":"STRING","value":"${target_platform}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"connection_options"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"param"},"2":{"slot":2,"type":"GRID","value":"value"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"connection_options"},"2":{"slot":2,"type":"STRING","value":"grid"}}},"2":{"slot":2,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"source_list"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"tableName"},"2":{"slot":2,"type":"GRID","value":"columnName"},"3":{"slot":3,"type":"GRID","value":"isIncremental"},"4":{"slot":4,"type":"GRID","value":"isKey"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"source_list"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot Incremental Load"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Incremental"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25570":{"id":25570,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1291941950,"x":-80,"y":-80,"width":32,"height":32,"inputConnectorIDs":[25577],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[25579],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Drop Stage Tables"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_database}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${metadata_stage_table_name}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Ignore Missing","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Yes"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25571":{"id":25571,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":-399,"y":161,"width":32,"height":32,"inputConnectorIDs":[25578],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Iteration Failure"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25572":{"id":25572,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":-240,"y":80,"width":32,"height":32,"inputConnectorIDs":[34431],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Failure"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"error"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Iteration Error"},"2":{"slot":2,"type":"STRING","value":"${iter_component_message ? iter_component_message : \" \"}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"Iteration Error At"},"2":{"slot":2,"type":"STRING","value":"${iter_component_name ? iter_component_name: \" \"}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"25573":{"id":25573,"sourceID":25569,"targetID":25566}},"failureConnectors":{"25578":{"id":25578,"sourceID":25569,"targetID":25571},"25579":{"id":25579,"sourceID":25570,"targetID":25567},"34431":{"id":34431,"sourceID":25569,"targetID":25572}},"unconditionalConnectors":{"25575":{"id":25575,"sourceID":25569,"targetID":25564},"25576":{"id":25576,"sourceID":25568,"targetID":25569},"25577":{"id":25577,"sourceID":25564,"targetID":25570}},"trueConnectors":{},"falseConnectors":{},"iterationConnectors":{},"noteConnectors":{},"canUndo":false,"undoCommand":"","undoCreated":0,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"25563":{"id":25563,"x":-795,"y":-398,"width":952,"height":635,"text":"**Execute ETL Flow for specific source table**\n\nThis job wraps our Stage 3 job (Stage Object). This is done so that we can manage some exported variables from the Stage 3. If we were directly iterating over (Stage Object) we would not be able to handle these exported variables.\n\n- (__Stage And Load__) Passes in the parameters and their values into the Stage - 3 job. Regardless of the result, the __metadata_stage_table_name__ and __stage_table_name__ are exported - this is to ensure these tables are dropped if the process fails before it has time to clean up these staging tables (when a run succeeds, these staging tables will no longer exist).\n\n- (__Drop Stage Tables__) Passes in the exported table names to be deleted. This flow is executed regardless of the result of Stage Object.","colour":"e6e63c"}},"variables":{"oauth":{"definition":{"name":"oauth","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"autoDebug":{"definition":{"name":"autoDebug","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"testLimit":{"definition":{"name":"testLimit","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"dist_style":{"definition":{"name":"dist_style","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"iter_state":{"definition":{"name":"iter_state","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"kms_key_id":{"definition":{"name":"kms_key_id","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"table_name":{"definition":{"name":"table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"log_metrics":{"definition":{"name":"log_metrics","type":"TEXT","scope":"TASKBATCH","description":"TRUE or FALSE (default=TRUE).\nWhether or not to add useful information to the metrics store.\n","visibility":"PUBLIC"},"value":""},"stage_prefix":{"definition":{"name":"stage_prefix","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"stage_schema":{"definition":{"name":"stage_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"staging_type":{"definition":{"name":"staging_type","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"error_message":{"definition":{"name":"error_message","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"iter_duration":{"definition":{"name":"iter_duration","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"0"},"source_schema":{"definition":{"name":"source_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"stage_dataset":{"definition":{"name":"stage_dataset","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"stage_project":{"definition":{"name":"stage_project","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_prefix":{"definition":{"name":"target_prefix","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_schema":{"definition":{"name":"target_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"blob_container":{"definition":{"name":"blob_container","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"s3_bucket_name":{"definition":{"name":"s3_bucket_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"stage_database":{"definition":{"name":"stage_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_dataset":{"definition":{"name":"target_dataset","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_project":{"definition":{"name":"target_project","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"stage_warehouse":{"definition":{"name":"stage_warehouse","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"storage_account":{"definition":{"name":"storage_account","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_database":{"definition":{"name":"target_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_platform":{"definition":{"name":"target_platform","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_table_name":{"definition":{"name":"stage_table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"target_warehouse":{"definition":{"name":"target_warehouse","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"encryption_method":{"definition":{"name":"encryption_method","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_table_name":{"definition":{"name":"target_table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"autoDebugVerbosity":{"definition":{"name":"autoDebugVerbosity","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"cloud_storage_area":{"definition":{"name":"cloud_storage_area","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"iter_component_name":{"definition":{"name":"iter_component_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"limitedSQLStatement":{"definition":{"name":"limitedSQLStatement","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"iter_component_message":{"definition":{"name":"iter_component_message","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"password_manager_entry":{"definition":{"name":"password_manager_entry","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"metadata_stage_table_name":{"definition":{"name":"metadata_stage_table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""}},"grids":{"message":{"definition":{"name":"message","scope":"BRANCH","definitions":[{"name":"key","type":"TEXT"},{"name":"value","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[{"values":["Iteration Duration","${iter_duration}"]},{"values":["Iteration End","${iter_state}"]}]},"source_list":{"definition":{"name":"source_list","scope":"BRANCH","definitions":[{"name":"tableName","type":"TEXT"},{"name":"columnName","type":"TEXT"},{"name":"isIncremental","type":"DECIMAL"},{"name":"isKey","type":"DECIMAL"}],"description":"","visibility":"PUBLIC"},"values":[]},"connection_options":{"definition":{"name":"connection_options","scope":"BRANCH","definitions":[{"name":"param","type":"TEXT"},{"name":"value","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]}}},"HubSpot - 1 - Orchestrator":{"id":-2,"revision":1,"created":1702911318834,"timestamp":1702911318834,"components":{"25647":{"id":25647,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":3,"y":6,"width":32,"height":32,"inputConnectorIDs":[25663],"outputSuccessConnectorIDs":[25657],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Pipeline Start"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"info"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Pipeline Started"},"2":{"slot":2,"type":"STRING","value":" "}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25648":{"id":25648,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":480,"y":0,"width":32,"height":32,"inputConnectorIDs":[25660],"outputSuccessConnectorIDs":[25661],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Update [Environment Default]"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"# Script to replace any [Environment Default] entries with the schema name\nif target_platform == \"Redshift\":\n\n    if stage_schema == \"[Environment Default]\":\n        context.updateVariable(\"stage_schema\", environment_default_schema)\n        print(\n            f\"Updating stage_schema from [Environment Default] to: {environment_default_schema}\"\n        )\n\n    if target_schema == \"[Environment Default]\":\n        context.updateVariable(\"target_schema\", environment_default_schema)\n        print(\n            f\"Updating target_schema from [Environment Default] to: {environment_default_schema}\"\n        )\n\nelif target_platform == \"Snowflake\":\n\n    if target_warehouse == \"[Environment Default]\":\n        context.updateVariable(\"target_warehouse\", environment_port)\n        print(\n            f\"Updating target_warehouse from [Environment Default] to: {environment_port}\"\n        )\n\n    if target_database == \"[Environment Default]\":\n        context.updateVariable(\"target_database\", environment_database)\n        print(\n            f\"Updating target_database from [Environment Default] to: {environment_database}\"\n        )\n\n    if target_schema == \"[Environment Default]\":\n        context.updateVariable(\"target_schema\", environment_default_schema)\n        print(\n            f\"Updating target_schema from [Environment Default] to: {environment_default_schema}\"\n        )\n\n    if stage_warehouse == \"[Environment Default]\":\n        context.updateVariable(\"stage_warehouse\", environment_port)\n        print(\n            f\"Updating stage_warehouse from [Environment Default] to: {environment_port}\"\n        )\n\n    if stage_database == \"[Environment Default]\":\n        context.updateVariable(\"stage_database\", environment_database)\n        print(\n            f\"Updating stage_database from [Environment Default] to: {environment_database}\"\n        )\n\n    if stage_schema == \"[Environment Default]\":\n        context.updateVariable(\"stage_schema\", environment_default_schema)\n        print(\n            f\"Updating stage_schema from [Environment Default] to: {environment_default_schema}\"\n        )\n\nelif target_platform == \"BigQuery\":\n\n    if stage_dataset == \"[Environment Default]\":\n        context.updateVariable(\"stage_dataset\", environment_default_schema)\n        print(\n            f\"Updating stage_dataset from [Environment Default] to: {environment_default_schema}\"\n        )\n\n    if target_dataset == \"[Environment Default]\":\n        context.updateVariable(\"target_dataset\", environment_default_schema)\n        print(\n            f\"Updating target_dataset from [Environment Default] to: {environment_default_schema}\"\n        )\n\n    if stage_project == \"[Environment Default]\":\n        context.updateVariable(\"stage_project\", environment_database)\n        print(\n            f\"Updating stage_project from [Environment Default] to: {environment_default_schema}\"\n        )\n\n    if target_project == \"[Environment Default]\":\n        context.updateVariable(\"target_project\", environment_database)\n        print(\n            f\"Updating target_project from [Environment Default] to: {environment_default_schema}\"\n        )\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25649":{"id":25649,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":800,"y":0,"width":32,"height":32,"inputConnectorIDs":[25662],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Fin"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25650":{"id":25650,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":800,"y":80,"width":32,"height":32,"inputConnectorIDs":[25664],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Failure"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25651":{"id":25651,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":-160,"y":0,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[25663],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25652":{"id":25652,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186960,"x":320,"y":0,"width":32,"height":32,"inputConnectorIDs":[25659],"outputSuccessConnectorIDs":[25660],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Get Unique Table Names"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"tbls = list(set([str(x[0]) for x in context.getGridVariable('source_list')]))\nunique_tbls = [[x] for x in tbls]\n\n# set the Grid Variable \nprint(\"Updated GridVariable 'DistinctTables' with: {}\".format(unique_tbls))\ncontext.updateGridVariable(\"DistinctTables\",unique_tbls)\n\n\ninternal_source_list = [[x[0], x[1], '0', '0'] for x in context.getGridVariable('source_list')]\ncontext.updateGridVariable('internal_source_list', internal_source_list)\n\n\n\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25653":{"id":25653,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"ITERATE","implementationID":-424773870,"x":640,"y":4,"width":32,"height":16,"inputConnectorIDs":[25661],"outputSuccessConnectorIDs":[25662],"outputFailureConnectorIDs":[25664],"outputUnconditionalConnectorIDs":[40166],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"Duration","mapTo":"logging_duration","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"Status","mapTo":"status","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Iterate Through Tables"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Grid Variable","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DistinctTables"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Grid Variable Mapping","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"table_name"},"2":{"slot":2,"type":"STRING","value":"table_name"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Break on Failure","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"No"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Concurrency","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${load_concurrent}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Stop on Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"No"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":false,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Condition","elements":{},"visible":false,"status":{},"columnNames":[]},"10":{"slot":10,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":false,"status":{},"columnNames":[]},"11":{"slot":11,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"999":{"slot":999,"name":"Record Values In Task History","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Yes"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[25665],"inputIterationConnectorIDs":[]},"25654":{"id":25654,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":640,"y":4,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"2 - Wrapper"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot - 2 - Wrapper"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"table_name"},"2":{"slot":2,"type":"STRING","value":"${table_name}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"oauth"},"2":{"slot":2,"type":"STRING","value":"${oauth}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"s3_bucket_name"},"2":{"slot":2,"type":"STRING","value":"${s3_bucket_name}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"stage_prefix"},"2":{"slot":2,"type":"STRING","value":"${stage_prefix}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"stage_schema"},"2":{"slot":2,"type":"STRING","value":"${stage_schema}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"target_prefix"},"2":{"slot":2,"type":"STRING","value":"${target_prefix}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"target_schema"},"2":{"slot":2,"type":"STRING","value":"${target_schema}"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"dist_style"},"2":{"slot":2,"type":"STRING","value":"${dist_style}"}}},"9":{"slot":9,"values":{"1":{"slot":1,"type":"STRING","value":"target_platform"},"2":{"slot":2,"type":"STRING","value":"${target_platform}"}}},"10":{"slot":10,"values":{"1":{"slot":1,"type":"STRING","value":"stage_database"},"2":{"slot":2,"type":"STRING","value":"${stage_database}"}}},"11":{"slot":11,"values":{"1":{"slot":1,"type":"STRING","value":"stage_project"},"2":{"slot":2,"type":"STRING","value":"${stage_project}"}}},"12":{"slot":12,"values":{"1":{"slot":1,"type":"STRING","value":"stage_warehouse"},"2":{"slot":2,"type":"STRING","value":"${stage_warehouse}"}}},"13":{"slot":13,"values":{"1":{"slot":1,"type":"STRING","value":"stage_dataset"},"2":{"slot":2,"type":"STRING","value":"${stage_dataset}"}}},"14":{"slot":14,"values":{"1":{"slot":1,"type":"STRING","value":"cloud_storage_area"},"2":{"slot":2,"type":"STRING","value":"${cloud_storage_area}"}}},"15":{"slot":15,"values":{"1":{"slot":1,"type":"STRING","value":"target_database"},"2":{"slot":2,"type":"STRING","value":"${target_database}"}}},"16":{"slot":16,"values":{"1":{"slot":1,"type":"STRING","value":"target_dataset"},"2":{"slot":2,"type":"STRING","value":"${target_dataset}"}}},"17":{"slot":17,"values":{"1":{"slot":1,"type":"STRING","value":"target_project"},"2":{"slot":2,"type":"STRING","value":"${target_project}"}}},"18":{"slot":18,"values":{"1":{"slot":1,"type":"STRING","value":"target_warehouse"},"2":{"slot":2,"type":"STRING","value":"${target_warehouse}"}}},"19":{"slot":19,"values":{"1":{"slot":1,"type":"STRING","value":"blob_container"},"2":{"slot":2,"type":"STRING","value":"${blob_container}"}}},"20":{"slot":20,"values":{"1":{"slot":1,"type":"STRING","value":"encryption_method"},"2":{"slot":2,"type":"STRING","value":"${encryption_method}"}}},"21":{"slot":21,"values":{"1":{"slot":1,"type":"STRING","value":"kms_key_id"},"2":{"slot":2,"type":"STRING","value":"${kms_key_id}"}}},"22":{"slot":22,"values":{"1":{"slot":1,"type":"STRING","value":"staging_type"},"2":{"slot":2,"type":"STRING","value":"${staging_type}"}}},"23":{"slot":23,"values":{"1":{"slot":1,"type":"STRING","value":"autoDebug"},"2":{"slot":2,"type":"STRING","value":"${autoDebug}"}}},"24":{"slot":24,"values":{"1":{"slot":1,"type":"STRING","value":"autoDebugVerbosity"},"2":{"slot":2,"type":"STRING","value":"${autoDebugVerbosity}"}}},"25":{"slot":25,"values":{"1":{"slot":1,"type":"STRING","value":"testLimit"},"2":{"slot":2,"type":"STRING","value":"${testLimit}"}}},"26":{"slot":26,"values":{"1":{"slot":1,"type":"STRING","value":"storage_account"},"2":{"slot":2,"type":"STRING","value":"${storage_account}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"advanced_connection_options"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"parameter"},"2":{"slot":2,"type":"GRID","value":"value"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"connection_options"},"2":{"slot":2,"type":"STRING","value":"grid"}}},"2":{"slot":2,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"internal_source_list"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"tableName"},"2":{"slot":2,"type":"GRID","value":"columnName"},"3":{"slot":3,"type":"GRID","value":"incremental"},"4":{"slot":4,"type":"GRID","value":"key"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"source_list"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot Incremental Load"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Incremental"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[25665]},"25656":{"id":25656,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":156,"y":-7,"width":32,"height":32,"inputConnectorIDs":[25657],"outputSuccessConnectorIDs":[25659],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Magic Variables"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"# this may or may not contain variables which are not valid connection options\n# and are designed to be used elsewhere within the shared job, they need to be removed from the grid \n# to prevent downstream errors \nopts = context.getGridVariable('advanced_connection_options')\n\n# create a new list to store the real connection options \nfiltered = [] \n\n# magic vars is a map: lower case param name => job variable name \nmagic_vars = {\n\t'autodebug': 'autoDebug', # passed into XX Query Components for CData Debugging \n    'verbosity': 'autoDebugVerbosity', # passed into XX Query Components (default = 3)\n    'testlimit': 'testLimit', # HubSpot specific var for limiting the amount of rows returned \n}\n\nif opts: \n  for r in opts:\n    param = r[0]\n    val = r[1]\n    \n    p = param.lower()\n    # check if this is a magic variable \n    # update the underlying job variable if it is \n    if p in magic_vars:\n      context.updateVariable(magic_vars[p], val)\n    # otherwise, its a normal connection option \n    else:\n      filtered.append( [param, val] )\n    \n  context.updateGridVariable('advanced_connection_options', filtered)\n  \n  \n  "}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"40159":{"id":40159,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":800,"y":-80,"width":32,"height":32,"inputConnectorIDs":[40166],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Pipeline End"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"info"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Pipeline Duration"},"2":{"slot":2,"type":"STRING","value":"${logging_duration}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"Pipeline End"},"2":{"slot":2,"type":"STRING","value":"${status}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"25657":{"id":25657,"sourceID":25647,"targetID":25656},"25659":{"id":25659,"sourceID":25656,"targetID":25652},"25660":{"id":25660,"sourceID":25652,"targetID":25648},"25661":{"id":25661,"sourceID":25648,"targetID":25653},"25662":{"id":25662,"sourceID":25653,"targetID":25649}},"failureConnectors":{"25664":{"id":25664,"sourceID":25653,"targetID":25650}},"unconditionalConnectors":{"25663":{"id":25663,"sourceID":25651,"targetID":25647},"40166":{"id":40166,"sourceID":25653,"targetID":40159}},"trueConnectors":{},"falseConnectors":{},"iterationConnectors":{"25665":{"id":25665,"sourceID":25653,"targetID":25654}},"noteConnectors":{},"canUndo":false,"undoCommand":"","undoCreated":0,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"25643":{"id":25643,"x":1665,"y":-573,"width":250,"height":58,"text":"** IGNORE **","colour":"d60000"},"25644":{"id":25644,"x":-1389,"y":376,"width":250,"height":58,"text":"** IGNORE **","colour":"d60000"},"25645":{"id":25645,"x":-791,"y":-471,"width":624,"height":388,"text":"**HubSpot Incremental Load** \n\nRoot (Stage 1) Orchestration Job of the Matillion Incremental Load pattern. \nThis job is the entry point of the shared job and is responsible for validating and reformatting some of the supplied data.\nThe pattern aims to synchronize INSERTS and UPDATES in a particular source, to a table within the relevant Cloud Data Warehouse.\n\n**Pattern Overview**\n\nThere are 4 distinct stages, each stage is represented by an orchestration job using the naming convention:\n__**<Source Name> - <Stage Number> - <Stage Name>**__\n\n+ **Stage 1** -  performs input validation and prepares the variables for **Stage 3**\n\n+ **Stage 2** - wrapper for **Stage 3** to ensure that in the case of any errors, relevant staging tables are nicely cleaned up.\n\n+ **Stage 3** - Orchestration job containing the ETL logic for a single source table.\n\n+ **Stage 4** - wrapper orchestration job to Query the underlying source. This is abstracted away from the rest of the logic to make the pattern more reusable.\n\nMore detailed information and specifics relating to each connector can be found within each job.\n\n\n======================================================================================================================================================================================================","colour":"00ce4f"},"25646":{"id":25646,"x":-75,"y":-233,"width":789,"height":310,"text":"** Prepare User Input **\n\n- (__Log Pipeline Start__) Writes to the metrics table.\n\n- (__Magic Variables__) Script that extracts some special variables from connection options. These are used internally to allow us to use 'hidden' variables that will not affect the slots of the shared job. Currently, this is used to switch on CData logging.\n\n- (__Get Unique Table Names__) Script to extract a list of table names from the source. This list is then iterated over, and the same ETL logic is applied to each source. This information is stored in the __distinct_table_names__ grid variable.\n\n- (__Update [Environment Default]__) Script to replace any [Environment Default] values with their actual value.\n\n- (__2 - Wrapper__) Iterate over each of the __distinct_table_names__ and call the lower level ETL process for each table in the source.","colour":"e6e63c"}},"variables":{"oauth":{"definition":{"name":"oauth","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":null},"status":{"definition":{"name":"status","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"autoDebug":{"definition":{"name":"autoDebug","type":"TEXT","scope":"BRANCH","description":"","visibility":"PRIVATE"},"value":"Off"},"testLimit":{"definition":{"name":"testLimit","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"dist_style":{"definition":{"name":"dist_style","type":"TEXT","scope":"BRANCH","description":"The distribution style to be used for both the staged and target data.\nThe options are:\nEVEN: Distribute data across all nodes evenly.\nALL: Distribute a full copy of the data onto each node.\nNote: KEY distribution is not implemented.","visibility":"PRIVATE"},"value":null},"kms_key_id":{"definition":{"name":"kms_key_id","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":null},"table_name":{"definition":{"name":"table_name","type":"TEXT","scope":"BRANCH","description":"","visibility":"PRIVATE"},"value":""},"stage_prefix":{"definition":{"name":"stage_prefix","type":"TEXT","scope":"BRANCH","description":"The prefix to assign to each of the source tables when they are staged. This is user assigned to help avoid any potential clashes with existing tables.\nE.g. If stage_prefix is set to 'stage_' and the table being processed is named 'test_data' then the staged table will be stored as 'stage_test_data'.","visibility":"PUBLIC"},"value":null},"stage_schema":{"definition":{"name":"stage_schema","type":"TEXT","scope":"BRANCH","description":"The schema where the staged data will be stored.","visibility":"PUBLIC"},"value":null},"staging_type":{"definition":{"name":"staging_type","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"Snowflake Managed"},"stage_dataset":{"definition":{"name":"stage_dataset","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":null},"stage_project":{"definition":{"name":"stage_project","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"target_prefix":{"definition":{"name":"target_prefix","type":"TEXT","scope":"BRANCH","description":"The prefix to assign to each of the target tables when they are created/updated. This is user assigned to help avoid any potential clashes with existing tables.\nE.g. If target_prefix is set to 'target_' and the table being processed is named 'test_data' then the staged table will be stored as 'target_test_data'.","visibility":"PUBLIC"},"value":null},"target_schema":{"definition":{"name":"target_schema","type":"TEXT","scope":"BRANCH","description":"The schema where the target data will be stored.","visibility":"PUBLIC"},"value":null},"blob_container":{"definition":{"name":"blob_container","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"s3_bucket_name":{"definition":{"name":"s3_bucket_name","type":"TEXT","scope":"BRANCH","description":"The name of the AWS S3 bucket that will be used for staging the source data.","visibility":"PUBLIC"},"value":null},"stage_database":{"definition":{"name":"stage_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_dataset":{"definition":{"name":"target_dataset","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"target_project":{"definition":{"name":"target_project","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"load_concurrent":{"definition":{"name":"load_concurrent","type":"TEXT","scope":"BRANCH","description":"Determines whether the tables being processed should be processed one after the other (Sequential) or at the same time (Concurrent).","visibility":"PUBLIC"},"value":"Sequential"},"stage_warehouse":{"definition":{"name":"stage_warehouse","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"storage_account":{"definition":{"name":"storage_account","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_database":{"definition":{"name":"target_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":null},"target_platform":{"definition":{"name":"target_platform","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"Snowflake"},"logging_duration":{"definition":{"name":"logging_duration","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"target_warehouse":{"definition":{"name":"target_warehouse","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"encryption_method":{"definition":{"name":"encryption_method","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"None"},"autoDebugVerbosity":{"definition":{"name":"autoDebugVerbosity","type":"TEXT","scope":"BRANCH","description":"","visibility":"PRIVATE"},"value":"3"},"cloud_storage_area":{"definition":{"name":"cloud_storage_area","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":null},"password_manager_entry":{"definition":{"name":"password_manager_entry","type":"TEXT","scope":"BRANCH","description":"The entry name in the password manager to use for the source database connection. This will need to be set up separately from the shared job.\nNote: This applies to the Generic JDBC routine. Different authentication options may be required for the other routines.","visibility":"PRIVATE"},"value":""}},"grids":{"source_list":{"definition":{"name":"source_list","scope":"BRANCH","definitions":[{"name":"table","type":"TEXT"},{"name":"column","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]},"DistinctTables":{"definition":{"name":"DistinctTables","scope":"BRANCH","definitions":[{"name":"table_name","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[{"values":["HubSpotDummyTableIgnoreThis"]}]},"internal_source_list":{"definition":{"name":"internal_source_list","scope":"BRANCH","definitions":[{"name":"tableName","type":"TEXT"},{"name":"columnName","type":"TEXT"},{"name":"incremental","type":"DECIMAL"},{"name":"key","type":"DECIMAL"}],"description":"","visibility":"PRIVATE"},"values":[]},"advanced_connection_options":{"definition":{"name":"advanced_connection_options","scope":"BRANCH","definitions":[{"name":"parameter","type":"TEXT"},{"name":"value","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]}}},"HubSpot - 3 - Stage and Load":{"id":-2,"revision":3,"created":1702996857112,"timestamp":1702996857112,"components":{"25588":{"id":25588,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":2000,"y":0,"width":32,"height":32,"inputConnectorIDs":[25611],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[25622],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Extract Start"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"info"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Extract Start"},"2":{"slot":2,"type":"STRING","value":"${load_strategy}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25589":{"id":25589,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":2160,"y":-160,"width":32,"height":32,"inputConnectorIDs":[25624],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Extract End"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"info"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Extract Stage Duration"},"2":{"slot":2,"type":"STRING","value":"${time_taken_to_stage ? time_taken_to_stage : 0}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"Extract Load Duration"},"2":{"slot":2,"type":"STRING","value":"${time_taken_to_load ? time_taken_to_load : 0}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"Extract Row Count"},"2":{"slot":2,"type":"STRING","value":"${static_row_count}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"Extract End"},"2":{"slot":2,"type":"STRING","value":"${extract_state}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"Extract Duration"},"2":{"slot":2,"type":"STRING","value":"${stage_duration}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25590":{"id":25590,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":2160,"y":160,"width":32,"height":32,"inputConnectorIDs":[34314],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End - Extract Failure"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25591":{"id":25591,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186960,"x":880,"y":0,"width":32,"height":32,"inputConnectorIDs":[25609],"outputSuccessConnectorIDs":[25613],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"Component","mapTo":"comp_name","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"Message","mapTo":"comp_message","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Adjust Source List"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"def truncate(word, length = 12):\n  \"\"\"\n  `truncate()` will convert items to a string and ensure they are the length specified by user \n  Used within stringer function to ensure all columns are the same width \n  \"\"\"\n  if len(word.strip()) <= length:\n    return word\n  return \"{}...\".format( word[:(length-3)] )\n  \ndef stringer(lst, colwidth = 30):\n  \"\"\"\n  `stringer()` is used for printing Grid Variables in a more readable manner.\n  \n  It will handle formatting the grid variable to look more like how it is presented in the product \n  (as a table) \n  \"\"\"\n  msgs = [] \n  for row in lst:\n    msg = \" | \".join(\n    \t[\n        \"{:<{width}}\".format( truncate(str(x), length = colwidth), width = colwidth)\n        for x in row \n        ]\n    )\n    msgs.append(msg)\n    \n  return \"\\n\" + \"\\n\".join(msgs)\n\nsource_list = context.getGridVariable('source_list')\nprint(\"Grid variable 'source_list': {}\".format(stringer(source_list)))\n        \nsource_table_metadata = context.getGridVariable('source_table_metadata')\nprint(\"\\nGrid variable 'source_table_metadata': {}\".format(stringer(source_table_metadata)))\n\ninc_col_options = ['Last Modified Date', 'Last modified date', 'CreatedAt', 'TimeAddedToList', 'SavedAt']\nprint(\"\\nVariable 'inc_col_options': {}\".format(inc_col_options))\n\n# is_table() checks that a row of a grid variable belongs to the table we \n# are currently iterating over\nis_table = lambda x: x[0] == table_name\n\n# filter source_list to current table and extract list of columns\nsrc = [ x for x in source_list if is_table(x) ] \nrequested_columns = [ x[1] for x in src ]\n\n# extract key and inc columns\n# Note that the true/false values for IsKey are sometimes truncated to three letters, i.e. tru/fal\nkey_columns = [ x[0] for x in source_table_metadata if x[6].lower().startswith('t')]\n\n# loop through ordered list of possible incremental columns, using the first one found\nmetadata_columns = [ x[0] for x in source_table_metadata ]\ninc_columns = []\nfor i in inc_col_options:\n  if i in metadata_columns:\n    inc_columns = [i]\n    break\n\n# add missing columns, preserving order set by user\nall_columns = requested_columns\n\nfor x in key_columns:\n    if x not in all_columns:\n        all_columns.append(x)\n\nfor x in inc_columns:\n    if x not in all_columns:\n        all_columns.append(x)   \n        \n# build new source column list with inc and key col flags\nnew_src = [] \n\nfor c in all_columns:\n    new_src.append(\n        [\n            table_name, \n            c,\n            '1' if c in inc_columns else '0',\n            '1' if c in key_columns else '0'\n  \t\t]\n  \t)\n    \ncontext.updateGridVariable('source_list', new_src)\nprint(\"\\nUpdated grid variable 'source_list': {}\".format(stringer(new_src)))\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25592":{"id":25592,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":0,"y":0,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[25623],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Start","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25593":{"id":25593,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":480,"y":0,"width":32,"height":32,"inputConnectorIDs":[25621],"outputSuccessConnectorIDs":[25610],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"Component","mapTo":"comp_name","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"error_message","mapTo":"comp_message","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Get Source Metadata"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot - 4 - Query Source"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"oauth"},"2":{"slot":2,"type":"STRING","value":"${oauth}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"sql"},"2":{"slot":2,"type":"STRING","value":"${source_metadata_sql}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"table"},"2":{"slot":2,"type":"STRING","value":"${metadata_stage_table_name}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"warehouse"},"2":{"slot":2,"type":"STRING","value":"${stage_warehouse}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"database"},"2":{"slot":2,"type":"STRING","value":"${stage_database}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"schema"},"2":{"slot":2,"type":"STRING","value":"${stage_schema}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"project"},"2":{"slot":2,"type":"STRING","value":"${stage_project}"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"dataset"},"2":{"slot":2,"type":"STRING","value":"${stage_dataset}"}}},"9":{"slot":9,"values":{"1":{"slot":1,"type":"STRING","value":"s3_bucket_name"},"2":{"slot":2,"type":"STRING","value":"${s3_bucket_name}"}}},"10":{"slot":10,"values":{"1":{"slot":1,"type":"STRING","value":"dist_style"},"2":{"slot":2,"type":"STRING","value":"${dist_style}"}}},"11":{"slot":11,"values":{"1":{"slot":1,"type":"STRING","value":"encryption_method"},"2":{"slot":2,"type":"STRING","value":"${encryption_method}"}}},"12":{"slot":12,"values":{"1":{"slot":1,"type":"STRING","value":"kms_key_id"},"2":{"slot":2,"type":"STRING","value":"${kms_key_id}"}}},"13":{"slot":13,"values":{"1":{"slot":1,"type":"STRING","value":"cloud_storage_area"},"2":{"slot":2,"type":"STRING","value":"${cloud_storage_area}"}}},"14":{"slot":14,"values":{"1":{"slot":1,"type":"STRING","value":"blob_container"},"2":{"slot":2,"type":"STRING","value":"${blob_container}"}}},"15":{"slot":15,"values":{"1":{"slot":1,"type":"STRING","value":"storage_account"},"2":{"slot":2,"type":"STRING","value":"${storage_account}"}}},"16":{"slot":16,"values":{"1":{"slot":1,"type":"STRING","value":"target_table_name"},"2":{"slot":2,"type":"STRING","value":"${target_table_name}"}}},"17":{"slot":17,"values":{"1":{"slot":1,"type":"STRING","value":"autoDebug"},"2":{"slot":2,"type":"STRING","value":"${autoDebug}"}}},"18":{"slot":18,"values":{"1":{"slot":1,"type":"STRING","value":"autoDebugVerbosity"},"2":{"slot":2,"type":"STRING","value":"${autoDebugVerbosity}"}}},"19":{"slot":19,"values":{"1":{"slot":1,"type":"STRING","value":"staging_type"},"2":{"slot":2,"type":"STRING","value":"${staging_type}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"connection_options"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"param"},"2":{"slot":2,"type":"GRID","value":"value"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"connectionOptions"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot Incremental Load"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Incremental"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25594":{"id":25594,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":1010676649,"x":160,"y":0,"width":32,"height":32,"inputConnectorIDs":[25623],"outputSuccessConnectorIDs":[25619],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"metadata_stage_table_name","mapTo":"metadata_stage_table_name","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"Component","mapTo":"component_name","validationStatus":null,"validationMessage":null},"3":{"slot":3,"fromId":null,"fromName":"source_metadata_sql","mapTo":"source_metadata_sql","validationStatus":null,"validationMessage":null},"4":{"slot":4,"fromId":null,"fromName":"get_metadata_sql","mapTo":"get_metadata_sql","validationStatus":null,"validationMessage":null},"5":{"slot":5,"fromId":null,"fromName":"stage_table_name","mapTo":"stage_table_name","validationStatus":null,"validationMessage":null},"6":{"slot":6,"fromId":null,"fromName":"target_table_name","mapTo":"target_table_name","validationStatus":null,"validationMessage":null}},"gridExportMappings":{"1":{"slot":1,"fromGrid":"source_list","toGrid":"source_list","columnMapToFrom":{"isKey":"key_column","isIncremental":"incremental_column","tableName":"table","columnName":"column"},"validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Build Strings"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"stage_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_database}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"stage_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"stage_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_project}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"stage_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"stage_prefix","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_prefix}"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"target_prefix","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_prefix}"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"metadata_stage_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"10":{"slot":10,"name":"stage_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"11":{"slot":11,"name":"target_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"12":{"slot":12,"name":"source_metadata_sql","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"13":{"slot":13,"name":"get_metadata_sql","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"14":{"slot":14,"name":"source_list","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"source_list"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"tableName"},"2":{"slot":2,"type":"GRID","value":"columnName"},"3":{"slot":3,"type":"GRID","value":"isIncremental"},"4":{"slot":4,"type":"GRID","value":"isKey"}}}},"visible":true,"status":{},"columnNames":[]},"15":{"slot":15,"name":"random_suffix","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25595":{"id":25595,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1096987946,"x":640,"y":0,"width":32,"height":32,"inputConnectorIDs":[25610],"outputSuccessConnectorIDs":[25609],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"comp_message","mapTo":"comp_message","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"comp_name","mapTo":"comp_name","validationStatus":null,"validationMessage":null}},"gridExportMappings":{"1":{"slot":1,"fromGrid":"source_table_metadata","toGrid":"source_table_metadata","columnMapToFrom":{"nullable":"is_nullable","dataTypeName":"data_type_name","isKey":"is_key","length":"length","numericScale":"numeric_scale","ordinal":"ordinal","columnName":"column_name"},"validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Check Source Metadata"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"metadata_stage_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${metadata_stage_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"target_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"get_metadata_sql","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${get_metadata_sql}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"stage_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_database}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"stage_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"stage_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_project}"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"stage_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"source_table_metadata","elements":{},"visible":true,"status":{},"columnNames":[]},"10":{"slot":10,"name":"error_message","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"11":{"slot":11,"name":"component_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"12":{"slot":12,"name":"component_message","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25596":{"id":25596,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-231597779,"x":1040,"y":0,"width":32,"height":32,"inputConnectorIDs":[25613],"outputSuccessConnectorIDs":[25616],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"Component","mapTo":"comp_name","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"error_message","mapTo":"comp_message","validationStatus":null,"validationMessage":null}},"gridExportMappings":{"1":{"slot":1,"fromGrid":"required_columns","toGrid":"source_list","columnMapToFrom":{"isKey":"key_column","isIncremental":"incremental_column","tableName":"table","columnName":"column_name"},"validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Check Required Columns"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"target_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"actual_columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"source_table_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"columnName"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"required_columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"source_list"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"tableName"},"2":{"slot":2,"type":"GRID","value":"columnName"},"3":{"slot":3,"type":"GRID","value":"isIncremental"},"4":{"slot":4,"type":"GRID","value":"isKey"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"missing_columns","elements":{},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"error_message","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25597":{"id":25597,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":947479920,"x":1200,"y":0,"width":32,"height":32,"inputConnectorIDs":[25616],"outputSuccessConnectorIDs":[25614],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"incremental_column","mapTo":"incremental_column","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"incremental_column_count","mapTo":"incremental_column_count","validationStatus":null,"validationMessage":null},"3":{"slot":3,"fromId":null,"fromName":"SELECT_STRING","mapTo":"source_column_list","validationStatus":null,"validationMessage":null},"4":{"slot":4,"fromId":null,"fromName":"key_column_count","mapTo":"key_column_count","validationStatus":null,"validationMessage":null}},"gridExportMappings":{"1":{"slot":1,"fromGrid":"key_columns","toGrid":"key_columns","columnMapToFrom":{"columnName":"column_name"},"validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Parameter Parser"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"platform","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_platform}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"incremental_source_list","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"source_list"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"tableName"},"2":{"slot":2,"type":"GRID","value":"columnName"},"3":{"slot":3,"type":"GRID","value":"isIncremental"},"4":{"slot":4,"type":"GRID","value":"isKey"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"source_list","elements":{},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"stage_warehouse","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_warehouse}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"stage_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_database}"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"stage_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"stage_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_project}"}}}},"visible":true,"status":{},"columnNames":[]},"10":{"slot":10,"name":"stage_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"11":{"slot":11,"name":"stage_prefix","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_prefix}"}}}},"visible":true,"status":{},"columnNames":[]},"12":{"slot":12,"name":"target_warehouse","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_warehouse}"}}}},"visible":true,"status":{},"columnNames":[]},"13":{"slot":13,"name":"target_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"14":{"slot":14,"name":"target_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"15":{"slot":15,"name":"target_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_project}"}}}},"visible":true,"status":{},"columnNames":[]},"16":{"slot":16,"name":"target_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"17":{"slot":17,"name":"target_prefix","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_prefix}"}}}},"visible":true,"status":{},"columnNames":[]},"18":{"slot":18,"name":"maximum_incremental_columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"DECIMAL","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"19":{"slot":19,"name":"source_enclosing_char_e","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"\""}}}},"visible":true,"status":{},"columnNames":[]},"20":{"slot":20,"name":"source_enclosing_char_s","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"\""}}}},"visible":true,"status":{},"columnNames":[]},"21":{"slot":21,"name":"metadata_stage_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${metadata_stage_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"22":{"slot":22,"name":"stage_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"23":{"slot":23,"name":"target_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"24":{"slot":24,"name":"column_count","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"DECIMAL","value":"0"}}}},"visible":true,"status":{},"columnNames":[]},"25":{"slot":25,"name":"key_column_count","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"DECIMAL","value":"0"}}}},"visible":true,"status":{},"columnNames":[]},"26":{"slot":26,"name":"key_columns","elements":{},"visible":true,"status":{},"columnNames":[]},"27":{"slot":27,"name":"incremental_column_count","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"DECIMAL","value":"0"}}}},"visible":true,"status":{},"columnNames":[]},"28":{"slot":28,"name":"incremental_column","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"29":{"slot":29,"name":"SELECT_STRING","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"30":{"slot":30,"name":"stage_table_metadata_sql","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25598":{"id":25598,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":1440,"y":0,"width":32,"height":32,"inputConnectorIDs":[25614],"outputSuccessConnectorIDs":[25612],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Build stage_single_sql"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"# set base stage_single_sql\nstage_single_sql = (f'SELECT {source_column_list} \\n' + \n                    f'FROM \"{table_name}\" \\n'\n                    f'LIMIT 1'\n)\n\ncontext.updateVariable('stage_single_sql', stage_single_sql)\nprint(f\"Updated Variable 'stage_single_sql' with {stage_single_sql}\")\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25599":{"id":25599,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":1600,"y":0,"width":32,"height":32,"inputConnectorIDs":[25612],"outputSuccessConnectorIDs":[25615],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"Component","mapTo":"comp_name","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"error_message","mapTo":"comp_message","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Stage Single Row"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot - 4 - Query Source"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"oauth"},"2":{"slot":2,"type":"STRING","value":"${oauth}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"sql"},"2":{"slot":2,"type":"STRING","value":"${stage_single_sql}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"table"},"2":{"slot":2,"type":"STRING","value":"${stage_table_name}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"warehouse"},"2":{"slot":2,"type":"STRING","value":"${stage_warehouse}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"database"},"2":{"slot":2,"type":"STRING","value":"${stage_database}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"schema"},"2":{"slot":2,"type":"STRING","value":"${stage_schema}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"project"},"2":{"slot":2,"type":"STRING","value":"${stage_project}"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"dataset"},"2":{"slot":2,"type":"STRING","value":"${stage_dataset}"}}},"9":{"slot":9,"values":{"1":{"slot":1,"type":"STRING","value":"s3_bucket_name"},"2":{"slot":2,"type":"STRING","value":"${s3_bucket_name}"}}},"10":{"slot":10,"values":{"1":{"slot":1,"type":"STRING","value":"dist_style"},"2":{"slot":2,"type":"STRING","value":"${dist_style}"}}},"11":{"slot":11,"values":{"1":{"slot":1,"type":"STRING","value":"encryption_method"},"2":{"slot":2,"type":"STRING","value":"${encryption_method}"}}},"12":{"slot":12,"values":{"1":{"slot":1,"type":"STRING","value":"kms_key_id"},"2":{"slot":2,"type":"STRING","value":"${kms_key_id}"}}},"13":{"slot":13,"values":{"1":{"slot":1,"type":"STRING","value":"cloud_storage_area"},"2":{"slot":2,"type":"STRING","value":"${cloud_storage_area}"}}},"14":{"slot":14,"values":{"1":{"slot":1,"type":"STRING","value":"blob_container"},"2":{"slot":2,"type":"STRING","value":"${blob_container}"}}},"15":{"slot":15,"values":{"1":{"slot":1,"type":"STRING","value":"storage_account"},"2":{"slot":2,"type":"STRING","value":"${storage_account}"}}},"16":{"slot":16,"values":{"1":{"slot":1,"type":"STRING","value":"target_table_name"},"2":{"slot":2,"type":"STRING","value":"${target_table_name}"}}},"17":{"slot":17,"values":{"1":{"slot":1,"type":"STRING","value":"autoDebug"},"2":{"slot":2,"type":"STRING","value":"${autoDebug}"}}},"18":{"slot":18,"values":{"1":{"slot":1,"type":"STRING","value":"autoDebugVerbosity"},"2":{"slot":2,"type":"STRING","value":"${autoDebugVerbosity}"}}},"19":{"slot":19,"values":{"1":{"slot":1,"type":"STRING","value":"staging_type"},"2":{"slot":2,"type":"STRING","value":"${staging_type}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"connection_options"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"param"},"2":{"slot":2,"type":"GRID","value":"value"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"connectionOptions"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot Incremental Load"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Incremental"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25601":{"id":25601,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":332733136,"x":1760,"y":0,"width":32,"height":32,"inputConnectorIDs":[25615],"outputSuccessConnectorIDs":[25611],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"load_strategy","mapTo":"load_strategy","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"comp_message","mapTo":"comp_message","validationStatus":null,"validationMessage":null},"3":{"slot":3,"fromId":null,"fromName":"comp_name","mapTo":"comp_name","validationStatus":null,"validationMessage":null},"4":{"slot":4,"fromId":null,"fromName":"stage_sql","mapTo":"stage_sql","validationStatus":null,"validationMessage":null},"5":{"slot":5,"fromId":null,"fromName":"inc_col_type","mapTo":"inc_col_type","validationStatus":null,"validationMessage":null},"6":{"slot":6,"fromId":null,"fromName":"max_value","mapTo":"max_value","validationStatus":null,"validationMessage":null},"7":{"slot":7,"fromId":null,"fromName":"incremental_column","mapTo":"incremental_column","validationStatus":null,"validationMessage":null}},"gridExportMappings":{"1":{"slot":1,"fromGrid":"stage_table_metadata","toGrid":"stage_table_metadata","columnMapToFrom":{"databaseType":"database_type","size":"size","type":"type","precision":"precision","columnName":"column_name"},"validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Decide Load Strategy"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"source_column_list","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${source_column_list}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"key_column_count","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${key_column_count}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"incremental_column_count","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${incremental_column_count}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"incremental_column","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${incremental_column}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"skip_inc_col_check","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"N"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"timestamp_precision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${timestamp_precision}"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"safely_cast_datetime_to_date","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"N"}}}},"visible":true,"status":{},"columnNames":[]},"10":{"slot":10,"name":"encloseTableString","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"TRUE"}}}},"visible":true,"status":{},"columnNames":[]},"11":{"slot":11,"name":"source_enclosing_char_s","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"\""}}}},"visible":true,"status":{},"columnNames":[]},"12":{"slot":12,"name":"source_enclosing_char_e","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"\""}}}},"visible":true,"status":{},"columnNames":[]},"13":{"slot":13,"name":"target_platform","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_platform}"}}}},"visible":true,"status":{},"columnNames":[]},"14":{"slot":14,"name":"stage_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_database}"}}}},"visible":true,"status":{},"columnNames":[]},"15":{"slot":15,"name":"stage_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"16":{"slot":16,"name":"stage_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_project}"}}}},"visible":true,"status":{},"columnNames":[]},"17":{"slot":17,"name":"stage_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"18":{"slot":18,"name":"stage_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"19":{"slot":19,"name":"target_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"20":{"slot":20,"name":"target_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"21":{"slot":21,"name":"target_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_project}"}}}},"visible":true,"status":{},"columnNames":[]},"22":{"slot":22,"name":"target_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"23":{"slot":23,"name":"target_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"24":{"slot":24,"name":"fix_target_data_types","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Yes"}}}},"visible":true,"status":{},"columnNames":[]},"25":{"slot":25,"name":"load_strategy","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"26":{"slot":26,"name":"max_value","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"27":{"slot":27,"name":"stage_sql","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"28":{"slot":28,"name":"inc_col_type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"29":{"slot":29,"name":"stage_table_metadata","elements":{},"visible":true,"status":{},"columnNames":[]},"30":{"slot":30,"name":"messages","elements":{},"visible":true,"status":{},"columnNames":[]},"31":{"slot":31,"name":"use_incremental_column","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Y"}}}},"visible":true,"status":{},"columnNames":[]},"32":{"slot":32,"name":"component_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"33":{"slot":33,"name":"component_message","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25602":{"id":25602,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":2160,"y":0,"width":32,"height":32,"inputConnectorIDs":[25622],"outputSuccessConnectorIDs":[25618],"outputFailureConnectorIDs":[34314],"outputUnconditionalConnectorIDs":[25624],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"row_count","mapTo":"static_row_count","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"Component","mapTo":"comp_name","validationStatus":null,"validationMessage":null},"3":{"slot":3,"fromId":null,"fromName":"error_message","mapTo":"comp_message","validationStatus":null,"validationMessage":null},"4":{"slot":4,"fromId":null,"fromName":"Status","mapTo":"extract_state","validationStatus":null,"validationMessage":null},"5":{"slot":5,"fromId":null,"fromName":"Duration","mapTo":"stage_duration","validationStatus":null,"validationMessage":null},"6":{"slot":6,"fromId":null,"fromName":"time_taken_to_load","mapTo":"time_taken_to_load","validationStatus":null,"validationMessage":null},"7":{"slot":7,"fromId":null,"fromName":"time_taken_to_stage","mapTo":"time_taken_to_stage","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Stage Data"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot - 4 - Query Source"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"oauth"},"2":{"slot":2,"type":"STRING","value":"${oauth}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"sql"},"2":{"slot":2,"type":"STRING","value":"${stage_sql}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"table"},"2":{"slot":2,"type":"STRING","value":"${stage_table_name}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"warehouse"},"2":{"slot":2,"type":"STRING","value":"${stage_warehouse}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"database"},"2":{"slot":2,"type":"STRING","value":"${stage_database}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"schema"},"2":{"slot":2,"type":"STRING","value":"${stage_schema}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"project"},"2":{"slot":2,"type":"STRING","value":"${stage_project}"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"dataset"},"2":{"slot":2,"type":"STRING","value":"${stage_dataset}"}}},"9":{"slot":9,"values":{"1":{"slot":1,"type":"STRING","value":"s3_bucket_name"},"2":{"slot":2,"type":"STRING","value":"${s3_bucket_name}"}}},"10":{"slot":10,"values":{"1":{"slot":1,"type":"STRING","value":"dist_style"},"2":{"slot":2,"type":"STRING","value":"${dist_style}"}}},"11":{"slot":11,"values":{"1":{"slot":1,"type":"STRING","value":"encryption_method"},"2":{"slot":2,"type":"STRING","value":"${encryption_method}"}}},"12":{"slot":12,"values":{"1":{"slot":1,"type":"STRING","value":"kms_key_id"},"2":{"slot":2,"type":"STRING","value":"${kms_key_id}"}}},"13":{"slot":13,"values":{"1":{"slot":1,"type":"STRING","value":"cloud_storage_area"},"2":{"slot":2,"type":"STRING","value":"${cloud_storage_area}"}}},"14":{"slot":14,"values":{"1":{"slot":1,"type":"STRING","value":"blob_container"},"2":{"slot":2,"type":"STRING","value":"${blob_container}"}}},"15":{"slot":15,"values":{"1":{"slot":1,"type":"STRING","value":"storage_account"},"2":{"slot":2,"type":"STRING","value":"${storage_account}"}}},"16":{"slot":16,"values":{"1":{"slot":1,"type":"STRING","value":"target_table_name"},"2":{"slot":2,"type":"STRING","value":"${target_table_name}"}}},"17":{"slot":17,"values":{"1":{"slot":1,"type":"STRING","value":"autoDebug"},"2":{"slot":2,"type":"STRING","value":"${autoDebug}"}}},"18":{"slot":18,"values":{"1":{"slot":1,"type":"STRING","value":"autoDebugVerbosity"},"2":{"slot":2,"type":"STRING","value":"${autoDebugVerbosity}"}}},"19":{"slot":19,"values":{"1":{"slot":1,"type":"STRING","value":"staging_type"},"2":{"slot":2,"type":"STRING","value":"${staging_type}"}}},"20":{"slot":20,"values":{"1":{"slot":1,"type":"STRING","value":"dataLimit"},"2":{"slot":2,"type":"STRING","value":"${testLimit}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"connection_options"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"param"},"2":{"slot":2,"type":"GRID","value":"value"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"connectionOptions"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot Incremental Load"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Incremental"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25603":{"id":25603,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-313802845,"x":2480,"y":0,"width":32,"height":32,"inputConnectorIDs":[42485],"outputSuccessConnectorIDs":[25617],"outputFailureConnectorIDs":[42477],"outputUnconditionalConnectorIDs":[25620],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"Status","mapTo":"load_state","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"comp_message","mapTo":"comp_message","validationStatus":null,"validationMessage":null},"3":{"slot":3,"fromId":null,"fromName":"comp_name","mapTo":"comp_name","validationStatus":null,"validationMessage":null},"4":{"slot":4,"fromId":null,"fromName":"logged_rows_deleted","mapTo":"load_deleted","validationStatus":null,"validationMessage":null},"5":{"slot":5,"fromId":null,"fromName":"logged_rows_inserted","mapTo":"load_inserted","validationStatus":null,"validationMessage":null},"6":{"slot":6,"fromId":null,"fromName":"logged_rows_update","mapTo":"load_updated","validationStatus":null,"validationMessage":null},"7":{"slot":7,"fromId":null,"fromName":"Duration","mapTo":"load_duration","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Load Target"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"platform","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_platform}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"stage_warehouse","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_warehouse}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"stage_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_database}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"stage_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"stage_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_project}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"stage_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"stage_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"target_warehouse","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_warehouse}"}}}},"visible":true,"status":{},"columnNames":[]},"10":{"slot":10,"name":"target_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"11":{"slot":11,"name":"target_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"12":{"slot":12,"name":"target_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_project}"}}}},"visible":true,"status":{},"columnNames":[]},"13":{"slot":13,"name":"target_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"14":{"slot":14,"name":"target_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"15":{"slot":15,"name":"load_strategy","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${load_strategy}"}}}},"visible":true,"status":{},"columnNames":[]},"16":{"slot":16,"name":"dist_style","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${dist_style}"}}}},"visible":true,"status":{},"columnNames":[]},"17":{"slot":17,"name":"columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"stage_table_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"columnName"}}}},"visible":true,"status":{},"columnNames":[]},"18":{"slot":18,"name":"key_columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"key_columns"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"columnName"}}}},"visible":true,"status":{},"columnNames":[]},"19":{"slot":19,"name":"incremental_column","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${incremental_column}"}}}},"visible":true,"status":{},"columnNames":[]},"20":{"slot":20,"name":"join_statement","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"21":{"slot":21,"name":"log_metrics","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"TRUE"}}}},"visible":true,"status":{},"columnNames":[]},"22":{"slot":22,"name":"static_row_count","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${static_row_count}"}}}},"visible":true,"status":{},"columnNames":[]},"23":{"slot":23,"name":"logged_rows_inserted","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"DECIMAL","value":"0"}}}},"visible":true,"status":{},"columnNames":[]},"24":{"slot":24,"name":"logged_rows_update","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"DECIMAL","value":"0"}}}},"visible":true,"status":{},"columnNames":[]},"25":{"slot":25,"name":"logged_rows_deleted","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"DECIMAL","value":"0"}}}},"visible":true,"status":{},"columnNames":[]},"26":{"slot":26,"name":"sub_RangeStartDate","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"27":{"slot":27,"name":"index_type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Clustered Column Store"}}}},"visible":true,"status":{},"columnNames":[]},"28":{"slot":28,"name":"component_message","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"29":{"slot":29,"name":"component_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25604":{"id":25604,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":2640,"y":0,"width":32,"height":32,"inputConnectorIDs":[25617],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Success"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25605":{"id":25605,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":2320,"y":0,"width":32,"height":32,"inputConnectorIDs":[25618],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[42485],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Load Start"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"info"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Load Start"},"2":{"slot":2,"type":"STRING","value":"${load_strategy}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25606":{"id":25606,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":2480,"y":-160,"width":32,"height":32,"inputConnectorIDs":[25620],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Load End"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"info"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Load Inserted Row Count"},"2":{"slot":2,"type":"STRING","value":"${load_inserted}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"Load Updated Row Count"},"2":{"slot":2,"type":"STRING","value":"${load_updated}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"Load Deleted Row Count"},"2":{"slot":2,"type":"STRING","value":"${load_deleted}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"Load Duration"},"2":{"slot":2,"type":"STRING","value":"${load_duration}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"Load End"},"2":{"slot":2,"type":"STRING","value":"${load_state}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25607":{"id":25607,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":2640,"y":160,"width":32,"height":32,"inputConnectorIDs":[34309],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End - Load Failure"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"25608":{"id":25608,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":320,"y":0,"width":32,"height":32,"inputConnectorIDs":[25619],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[25621],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Start"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"info"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Iteration Started"},"2":{"slot":2,"type":"STRING","value":"${table_name}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"34261":{"id":34261,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":2480,"y":160,"width":32,"height":32,"inputConnectorIDs":[42477],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[34309],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Load Error"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"error"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Load Error"},"2":{"slot":2,"type":"STRING","value":"${comp_message}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"Load Error At"},"2":{"slot":2,"type":"STRING","value":"${comp_name}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"25609":{"id":25609,"sourceID":25595,"targetID":25591},"25610":{"id":25610,"sourceID":25593,"targetID":25595},"25611":{"id":25611,"sourceID":25601,"targetID":25588},"25612":{"id":25612,"sourceID":25598,"targetID":25599},"25613":{"id":25613,"sourceID":25591,"targetID":25596},"25614":{"id":25614,"sourceID":25597,"targetID":25598},"25615":{"id":25615,"sourceID":25599,"targetID":25601},"25616":{"id":25616,"sourceID":25596,"targetID":25597},"25617":{"id":25617,"sourceID":25603,"targetID":25604},"25618":{"id":25618,"sourceID":25602,"targetID":25605},"25619":{"id":25619,"sourceID":25594,"targetID":25608}},"failureConnectors":{"34314":{"id":34314,"sourceID":25602,"targetID":25590},"42477":{"id":42477,"sourceID":25603,"targetID":34261}},"unconditionalConnectors":{"25620":{"id":25620,"sourceID":25603,"targetID":25606},"25621":{"id":25621,"sourceID":25608,"targetID":25593},"25622":{"id":25622,"sourceID":25588,"targetID":25602},"25623":{"id":25623,"sourceID":25592,"targetID":25594},"25624":{"id":25624,"sourceID":25602,"targetID":25589},"34309":{"id":34309,"sourceID":34261,"targetID":25607},"42485":{"id":42485,"sourceID":25605,"targetID":25603}},"trueConnectors":{},"falseConnectors":{},"iterationConnectors":{},"noteConnectors":{},"canUndo":true,"undoCommand":"Set Parameter","undoCreated":1702916056811,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"25582":{"id":25582,"x":3920,"y":-1342,"width":250,"height":58,"text":"**IGNORE**","colour":"d60000"},"25583":{"id":25583,"x":-2114,"y":644,"width":250,"height":58,"text":"**IGNORE**","colour":"d60000"},"25584":{"id":25584,"x":85,"y":-241,"width":630,"height":318,"text":"** Source Validation **\n\n- (__Build Strings__) Generate generic job variables (stage/target table names etc. - see Export tab) \n\n- (__Get Source Metadata__) Query CData __sys_tablecolumns__ metadata table to get information about the source and store in CDW \n\n- (__Check Source Metadata__) Query table created above, ensure it is valid, export the metadata to a grid (see Grid Export tab)","colour":"e6e63c"},"25585":{"id":25585,"x":804,"y":-237,"width":472,"height":311,"text":"**Input/Target Validation**\n\n- (__Check Required Columns__) Contains logic to validate that the columns which have been requested still exist in the source\n\n- (__Adjust Source List__) Modifies the current __source_list__ grid variable. This will set the correct key and incremental columns (where applicable). \nNB: For HubSpot, the __'Last Modified Date'__,  __'Last modified date'__,  __'CreatedAt'__ column is used as an incremental column when present.\n\n- (__Parameter Parser__) Contains logic to generate some useful Grid/Job variables (e.g key_columns Grid, SELECT list. See Export/Grid Export tabs)","colour":"e6e63c"},"25586":{"id":25586,"x":1366,"y":-235,"width":466,"height":310,"text":"**Decide Load Strategy**\n\n- (__Build stage_single_sql__) Generates and sets stage_single_sql variable which is used to query 1 row from the source. This logic is in its own Python script so that we can log the query.\n\n- (__Stage Single Row__) Executes the __stage_single_sql__ query. This creates a table which will be identical to the stage table for the ETL process, but with only 1 row - the metadata of this table is used within Decide Load Strategy.\n\n- (__Decide Load Strategy__) Executes packaged up logic to determine the type of load this job is doing (UPDATE/APPEND INSERT/TRUNCATE INSERT). \nThis exports the __stage_sql__ variable which will be used to stage the dataset.\nThis exports the __load_strategy__ variable which will be used within Load Target","colour":"e6e63c"},"25587":{"id":25587,"x":1925,"y":-314,"width":631,"height":550,"text":"** Load Data **\n\n- (__Log Extract Start__) Writes metrics to indicate the Extraction phase has started\n\n- (__Stage Data__) Executes __stage_sql__ query and stores the data in the staging table \n\n- (__Load Target__) Executes logic to move the Stage Table Data into the Target Table using the desired __Load Strategy__","colour":"e6e63c"}},"variables":{"oauth":{"definition":{"name":"oauth","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"autoDebug":{"definition":{"name":"autoDebug","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"comp_name":{"definition":{"name":"comp_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"max_value":{"definition":{"name":"max_value","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"stage_sql":{"definition":{"name":"stage_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"testLimit":{"definition":{"name":"testLimit","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"dist_style":{"definition":{"name":"dist_style","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"kms_key_id":{"definition":{"name":"kms_key_id","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"load_state":{"definition":{"name":"load_state","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"table_name":{"definition":{"name":"table_name","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"log_metrics":{"definition":{"name":"log_metrics","type":"TEXT","scope":"BRANCH","description":"TRUE or FALSE (default=TRUE).\nWhether or not to add useful information to the metrics store.\n","visibility":"PUBLIC"},"value":""},"comp_message":{"definition":{"name":"comp_message","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"inc_col_type":{"definition":{"name":"inc_col_type","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"load_deleted":{"definition":{"name":"load_deleted","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"0"},"load_updated":{"definition":{"name":"load_updated","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"0"},"stage_prefix":{"definition":{"name":"stage_prefix","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_schema":{"definition":{"name":"stage_schema","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"staging_type":{"definition":{"name":"staging_type","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"extract_state":{"definition":{"name":"extract_state","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"load_duration":{"definition":{"name":"load_duration","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"load_inserted":{"definition":{"name":"load_inserted","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"0"},"load_strategy":{"definition":{"name":"load_strategy","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"TRUNCATE INSERT"},"stage_dataset":{"definition":{"name":"stage_dataset","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_project":{"definition":{"name":"stage_project","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_prefix":{"definition":{"name":"target_prefix","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_schema":{"definition":{"name":"target_schema","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"blob_container":{"definition":{"name":"blob_container","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"s3_bucket_name":{"definition":{"name":"s3_bucket_name","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_database":{"definition":{"name":"stage_database","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_duration":{"definition":{"name":"stage_duration","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"},"target_dataset":{"definition":{"name":"target_dataset","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_project":{"definition":{"name":"target_project","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_warehouse":{"definition":{"name":"stage_warehouse","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"storage_account":{"definition":{"name":"storage_account","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_database":{"definition":{"name":"target_database","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_platform":{"definition":{"name":"target_platform","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"extract_duration":{"definition":{"name":"extract_duration","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"get_metadata_sql":{"definition":{"name":"get_metadata_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"key_column_count":{"definition":{"name":"key_column_count","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"stage_single_sql":{"definition":{"name":"stage_single_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"stage_table_name":{"definition":{"name":"stage_table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"static_row_count":{"definition":{"name":"static_row_count","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"target_warehouse":{"definition":{"name":"target_warehouse","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"encryption_method":{"definition":{"name":"encryption_method","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_table_name":{"definition":{"name":"target_table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"target_table_name"},"autoDebugVerbosity":{"definition":{"name":"autoDebugVerbosity","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"cloud_storage_area":{"definition":{"name":"cloud_storage_area","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"incremental_column":{"definition":{"name":"incremental_column","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"source_column_list":{"definition":{"name":"source_column_list","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"time_taken_to_load":{"definition":{"name":"time_taken_to_load","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"},"limitedSQLStatement":{"definition":{"name":"limitedSQLStatement","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"source_metadata_sql":{"definition":{"name":"source_metadata_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"time_taken_to_stage":{"definition":{"name":"time_taken_to_stage","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"},"timestamp_precision":{"definition":{"name":"timestamp_precision","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"microseconds"},"incremental_column_count":{"definition":{"name":"incremental_column_count","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"0"},"metadata_stage_table_name":{"definition":{"name":"metadata_stage_table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""}},"grids":{"key_columns":{"definition":{"name":"key_columns","scope":"BRANCH","definitions":[{"name":"columnName","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[]},"source_list":{"definition":{"name":"source_list","scope":"BRANCH","definitions":[{"name":"tableName","type":"TEXT"},{"name":"columnName","type":"TEXT"},{"name":"isIncremental","type":"DECIMAL"},{"name":"isKey","type":"DECIMAL"}],"description":"","visibility":"PUBLIC"},"values":[]},"connection_options":{"definition":{"name":"connection_options","scope":"BRANCH","definitions":[{"name":"param","type":"TEXT"},{"name":"value","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]},"stage_table_metadata":{"definition":{"name":"stage_table_metadata","scope":"BRANCH","definitions":[{"name":"columnName","type":"TEXT"},{"name":"type","type":"TEXT"},{"name":"size","type":"DECIMAL"},{"name":"precision","type":"DECIMAL"},{"name":"databaseType","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[]},"source_table_metadata":{"definition":{"name":"source_table_metadata","scope":"BRANCH","definitions":[{"name":"columnName","type":"TEXT"},{"name":"dataTypeName","type":"TEXT"},{"name":"length","type":"TEXT"},{"name":"numericScale","type":"TEXT"},{"name":"nullable","type":"TEXT"},{"name":"ordinal","type":"TEXT"},{"name":"isKey","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[]}}}},"transformationJobs":{}}],"version":"Built","environment":"snowflake"}