{"objects":[{"metadata":{"identifier":{"packageName":"Matillion.Incremental","name":"HubSpot Incremental Load","revision":1,"type":"DYNAMIC"},"rootJobReference":{"name":"HubSpot - 1 - Orchestrator","type":"ORCHESTRATION","parameterMetadata":[{"slot":2,"variableName":"oauth","variableType":"SCALAR","displayName":"OAuth","description":"The name of the OAuth entry to be used from the Matillion OAuth Manager to connect to the source. These must be set up in advance, using Project --> Mangage OAuth.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":3,"variableName":"source_list","variableType":"GRID","displayName":"Tables and Columns","description":"Contains the list of tables and columns (and in some cases an incremental_column to specify which column the load should be incremented on [accepts values of 0 or 1]) to be processed. ","defaultValue":[{"values":{"1":""}}],"defaultValueType":null,"required":true},{"slot":4,"variableName":"advanced_connection_options","variableType":"GRID","displayName":"Connection Options","description":"A list of values and parameters. Parameters and their allowed values are database/driver specific. Referring to the data model will provide insight of what you could provide here.\n<br>\nThey are usually not required as sensible defaults are assumed.","defaultValue":[{"values":{"1":""}}],"defaultValueType":null,"required":false},{"slot":5,"variableName":"dist_style","variableType":"SCALAR","displayName":"Distribution Style","description":"<b>Even:</b> Distributes rows around the Redshift cluster evenly.<br>\n<b>All:</b> Copy rows to all nodes in the Redshift cluster.<br>\nTable distribution is critical to good performance. See the <a href=\"http://docs.aws.amazon.com/redshift/latest/dg/tutorial-tuning-tables-distribution.html\" target=\"_blank\">Amazon Redshift</a> documentation for more information.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":6,"variableName":"load_concurrent","variableType":"SCALAR","displayName":"Load Type","description":"Sequential - Iterations are done in sequence, waiting for each to complete before starting the next. \nThis is the default.\n<br>\nConcurrent - Iterations are run concurrently. This requires all \"Variables to Iterate\" to be defined as\n<a href=\"https://redshiftsupport.matillion.com/customer/portal/articles/2037630?b_id=8915#header1\" target=\"_blank\"><i>Copied</i> variables</a>, so that each iteration gets its own copy of the variable isolated from the same\nvariable being used by other concurrent executions.\n<span class=\"env rs sf\"><br><b>Note:</b> The maximum concurrency is limited by the number of available threads (2x the number of virtual cpus on your cloud instance).</span>","defaultValue":[{"values":{"1":"Sequential"}}],"defaultValueType":"TEXT","required":true},{"slot":7,"variableName":"s3_bucket_name","variableType":"SCALAR","displayName":"S3 Bucket Name","description":"<span class=\"env sf\">(AWS Only) </span>The name of an S3 bucket for temporary storage. Ensure your access credentials have S3 access and permission to write to the bucket. See <a href=\"https://redshiftsupport.matillion.com/customer/portal/articles/2054760\" target=\"_blank\">this document</a> for details on setting up access. The temporary objects created in this bucket will be removed again after the load completes, they are not kept.<br>\n<span class=\"env sf\">This property is available when using an Existing Amazon S3 Location for Staging.</span>","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":8,"variableName":"stage_schema","variableType":"SCALAR","displayName":"Stage Schema","description":"The schema name where the staging data will be stored.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":9,"variableName":"stage_prefix","variableType":"SCALAR","displayName":"Stage Prefix","description":"A prefix value that will be added to the start of the stage table names.\n<br>\ne.g. If a Stage Prefix of 'stage_' is specified and the table being processed is named 'test_data' then the target table will be named 'stage_test_data'.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":10,"variableName":"target_schema","variableType":"SCALAR","displayName":"Target Schema","description":"The schema name where the target data will be stored.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":11,"variableName":"target_prefix","variableType":"SCALAR","displayName":"Target Prefix","description":"A prefix value that will be added to the start of the target table names.\n<br>\ne.g. If a Target Prefix of 'target_' is specified and the table being processed is named 'test_data' then the target table will be named 'target_test_data'.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":false},{"slot":12,"variableName":"encryption_method","variableType":"SCALAR","displayName":"Encryption","description":"<span class=\"env sf\">(AWS Only) </span>Decide on how the files are encrypted inside the S3 Bucket.<span class=\"env sf\">This property is available when using an Existing Amazon S3 Location for Staging.</span><br>\n                <b>None:</b> No encryption.<br>\n                <b>SSE KMS:</b> Encrypt the data according to a key stored on <a href=\"https://aws.amazon.com/kms/\" target=\"_blank\">KMS</a>.<br>\n                <b>SSE S3:</b> Encrypt the data according to a key stored on an <a href=\"http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html\" target=\"_blank\">S3 bucket</a>","defaultValue":[{"values":{"1":"None"}}],"defaultValueType":"TEXT","required":true},{"slot":13,"variableName":"kms_key_id","variableType":"SCALAR","displayName":"KMS Key ID","description":"The ID of the KMS encryption key you have chosen to use in the 'Encryption' property.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":false}]},"orchestrationJobs":["HubSpot - 1 - Orchestrator","HubSpot - 2 - Wrapper","HubSpot - 3 - Stage and Load","HubSpot - 4 - Query Source"],"transformationJobs":[],"imageMetadata":{"componentIcon":"iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAADMUlEQVR42t1XXUgUURjdfqnIyCio3sR8KIrCh+ilngrqwZdi2b2za4nVYvRgFo1z70w1RokFERL0UhCRYEUgZBYRFm710INP/aEUYVKCZBaiGe76dWYd1zuza+6uk0UDHzPsne+ec88933dnfb5/8SK/f87fAdaDARJKFwkWR0TJUIpymkdT8sksW5BdkggUAzSGICk6slEDoEtJVx7auUPEmZoFgeBpF7gdyqaM5+BKfUq+rmzOMJmpaQkYyvbMF8GephIIVmSWfNy/Egm9aUgMWN744wqMmSe8BmBXocYjJJ9BfEhOxNlFikTmTekBzp7YOfGsPJB+Qv8yeOC+tKIoifCq31cSK7Hf/e5NaZrmbJAw7dKEpKyHeHDrjBGQynQXJu2zJx9B1VTNKIHE5EZpAdRol7bkFrZp8cQYq8W+v0p6QGctMGGEqvwLvSOBDpcw6gSJN1CjDvef6ftIopd0gcg2j7eEHUD8mBw0JSyCO7wlwZmWBqgT/eAmxlrHvOIY+0Lcv8LD6mDvpMljqI6D5PPNcvQVwV46GxM759HJGdrimrjG/Y6qqnnX+JGjrq366JH8yiGnvM4GZYFzzp9pmnYe402Od0223Iv9lw+vuCy9dQG8GUGIWC1Xh+/ywzQgwvYBV1rgQRUoZY5V8dA6eVwIUQzwr7jfecwr3tfzY3SWq9Qj9o6SWp7nQVNSipwE4HyooOt6iUyiQau8h/HRmFDoBq8kqNEfmeJQy6YXtMkkXvPyVkty7Hu+dXLit3BcsG/j44MiRKe4Rhjf41ElMCYTiIoKgtSjeP6ELRpM6RFo0ye5FrWMOX3wE6EN7g+YNhC4hL2epBN2kxEohEIPEBe8A08cz2wfnt92inIyIHHS7RPRZJWpaZrzAd4PBUK5fWoLdsU+DfuS4EZg7fh/iWGN7dS51gGzjcB0jTCmuC6q6mDG9VJ57geRRbkY7rZrVb3j4PJVXV29ESB9WOVz3HdbciN6ZRK5Ot79wfpisncNwygAaCNiyG5GVrT7XM0qW8e3OAkEL0+VA6nnArgG0Q1ShdNTQA2uxodIM8A/w3QNZIaWZJIH6YPTBv+vr1+2ZGgUV7vrNwAAAABJRU5ErkJggg==","smallIcon":"iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAABfklEQVR42mNggIL/DAyMDOSC/5WRif8rI378r4rYgM2g/w0OLP+rI4uAamJxGbAPKPkfjKsjZDHkq6NC4fI1MepYDIjyAtp+Dajg5v/K8Ev/a6JU0SzQAMp9AOK7/8uSeHF7JS2N9X9VVB9YcXWUP6ohEcv+V0VWEhcmVZFhQFs/Af3d/j80lPl/ZbQW0IATQFeu/F8WLUOcITXhmkBN1/9XRN4A0v/gYQAK7MrIeBJcAtQENgQYU1VR5UD+VyD+jTUwMQM3YgVyyFdXV1tcrkxZCRariqolbEB1+Faw4pJYbhC/oqJia29F8YN1lTn/v1THdBPjhUqQAZ8qY2dVVVWFna9JV31bGfPyaWX8/4bK8knEGBADMuBhZcL/45WpP4Hsv9DEtr2yomIJbo21kdrAxFUFVPwFZMib6tjOJRV5X/8Bo/J+VeLM+w0JHJWVlXn4Au8pNKDKYWLl5eX+QLwWiO8BNdsTcvpCoAGPQWkBTYoRaEBFYWEhJwO1AACRK/lksZaMlwAAAABJRU5ErkJggg=="},"description":"","helpHtml":"<div class=\"mtln\"><img src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAADMUlEQVR42t1XXUgUURjdfqnIyCio3sR8KIrCh+ilngrqwZdi2b2za4nVYvRgFo1z70w1RokFERL0UhCRYEUgZBYRFm710INP/aEUYVKCZBaiGe76dWYd1zuza+6uk0UDHzPsne+ec88933dnfb5/8SK/f87fAdaDARJKFwkWR0TJUIpymkdT8sksW5BdkggUAzSGICk6slEDoEtJVx7auUPEmZoFgeBpF7gdyqaM5+BKfUq+rmzOMJmpaQkYyvbMF8GephIIVmSWfNy/Egm9aUgMWN744wqMmSe8BmBXocYjJJ9BfEhOxNlFikTmTekBzp7YOfGsPJB+Qv8yeOC+tKIoifCq31cSK7Hf/e5NaZrmbJAw7dKEpKyHeHDrjBGQynQXJu2zJx9B1VTNKIHE5EZpAdRol7bkFrZp8cQYq8W+v0p6QGctMGGEqvwLvSOBDpcw6gSJN1CjDvef6ftIopd0gcg2j7eEHUD8mBw0JSyCO7wlwZmWBqgT/eAmxlrHvOIY+0Lcv8LD6mDvpMljqI6D5PPNcvQVwV46GxM759HJGdrimrjG/Y6qqnnX+JGjrq366JH8yiGnvM4GZYFzzp9pmnYe402Od0223Iv9lw+vuCy9dQG8GUGIWC1Xh+/ywzQgwvYBV1rgQRUoZY5V8dA6eVwIUQzwr7jfecwr3tfzY3SWq9Qj9o6SWp7nQVNSipwE4HyooOt6iUyiQau8h/HRmFDoBq8kqNEfmeJQy6YXtMkkXvPyVkty7Hu+dXLit3BcsG/j44MiRKe4Rhjf41ElMCYTiIoKgtSjeP6ELRpM6RFo0ye5FrWMOX3wE6EN7g+YNhC4hL2epBN2kxEohEIPEBe8A08cz2wfnt92inIyIHHS7RPRZJWpaZrzAd4PBUK5fWoLdsU+DfuS4EZg7fh/iWGN7dS51gGzjcB0jTCmuC6q6mDG9VJ57geRRbkY7rZrVb3j4PJVXV29ESB9WOVz3HdbciN6ZRK5Ot79wfpisncNwygAaCNiyG5GVrT7XM0qW8e3OAkEL0+VA6nnArgG0Q1ShdNTQA2uxodIM8A/w3QNZIaWZJIH6YPTBv+vr1+2ZGgUV7vrNwAAAABJRU5ErkJggg==\"><h3>HubSpot Incremental Load</h3><p style='white-space: pre-line; word-wrap: break-word;'></p><h3>Properties</h3><table><tbody><tr><th>Property</th><th>Type</th><th>Description</th></tr><tr><td>OAuth</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>The name of the OAuth entry to be used from the Matillion OAuth Manager to connect to the source. These must be set up in advance, using Project --> Mangage OAuth.</td></tr><tr><td>Tables and Columns</td><td>Grid</td><td style='white-space: pre-line; word-wrap: break-word;'>Contains the list of tables and columns (and in some cases an incremental_column to specify which column the load should be incremented on [accepts values of 0 or 1]) to be processed. </td></tr><tr><td>Connection Options</td><td>Grid</td><td style='white-space: pre-line; word-wrap: break-word;'>A list of values and parameters. Parameters and their allowed values are database/driver specific. Referring to the data model will provide insight of what you could provide here.\n<br>\nThey are usually not required as sensible defaults are assumed.</td></tr><tr><td>Distribution Style</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'><b>Even:</b> Distributes rows around the Redshift cluster evenly.<br>\n<b>All:</b> Copy rows to all nodes in the Redshift cluster.<br>\nTable distribution is critical to good performance. See the <a href=\"http://docs.aws.amazon.com/redshift/latest/dg/tutorial-tuning-tables-distribution.html\" target=\"_blank\">Amazon Redshift</a> documentation for more information.</td></tr><tr><td>Load Type</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>Sequential - Iterations are done in sequence, waiting for each to complete before starting the next. \nThis is the default.\n<br>\nConcurrent - Iterations are run concurrently. This requires all \"Variables to Iterate\" to be defined as\n<a href=\"https://redshiftsupport.matillion.com/customer/portal/articles/2037630?b_id=8915#header1\" target=\"_blank\"><i>Copied</i> variables</a>, so that each iteration gets its own copy of the variable isolated from the same\nvariable being used by other concurrent executions.\n<span class=\"env rs sf\"><br><b>Note:</b> The maximum concurrency is limited by the number of available threads (2x the number of virtual cpus on your cloud instance).</span></td></tr><tr><td>S3 Bucket Name</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'><span class=\"env sf\">(AWS Only) </span>The name of an S3 bucket for temporary storage. Ensure your access credentials have S3 access and permission to write to the bucket. See <a href=\"https://redshiftsupport.matillion.com/customer/portal/articles/2054760\" target=\"_blank\">this document</a> for details on setting up access. The temporary objects created in this bucket will be removed again after the load completes, they are not kept.<br>\n<span class=\"env sf\">This property is available when using an Existing Amazon S3 Location for Staging.</span></td></tr><tr><td>Stage Schema</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>The schema name where the staging data will be stored.</td></tr><tr><td>Stage Prefix</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>A prefix value that will be added to the start of the stage table names.\n<br>\ne.g. If a Stage Prefix of 'stage_' is specified and the table being processed is named 'test_data' then the target table will be named 'stage_test_data'.</td></tr><tr><td>Target Schema</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>The schema name where the target data will be stored.</td></tr><tr><td>Target Prefix</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>A prefix value that will be added to the start of the target table names.\n<br>\ne.g. If a Target Prefix of 'target_' is specified and the table being processed is named 'test_data' then the target table will be named 'target_test_data'.</td></tr><tr><td>Encryption</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'><span class=\"env sf\">(AWS Only) </span>Decide on how the files are encrypted inside the S3 Bucket.<span class=\"env sf\">This property is available when using an Existing Amazon S3 Location for Staging.</span><br>\n                <b>None:</b> No encryption.<br>\n                <b>SSE KMS:</b> Encrypt the data according to a key stored on <a href=\"https://aws.amazon.com/kms/\" target=\"_blank\">KMS</a>.<br>\n                <b>SSE S3:</b> Encrypt the data according to a key stored on an <a href=\"http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html\" target=\"_blank\">S3 bucket</a></td></tr><tr><td>KMS Key ID</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>The ID of the KMS encryption key you have chosen to use in the 'Encryption' property.</td></tr></tbody></table></div>","created":1702919321344},"orchestrationJobs":{"HubSpot - 4 - Query Source":{"id":-2,"revision":1,"created":1702911151121,"timestamp":1702911151121,"components":{"31256":{"id":31256,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":-160,"y":0,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[31266],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Start","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31257":{"id":31257,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":1847161714,"x":160,"y":0,"width":32,"height":32,"inputConnectorIDs":[31264],"outputSuccessConnectorIDs":[31263],"outputFailureConnectorIDs":[31268],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"Message","mapTo":"error_message","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"Time Taken To Load","mapTo":"time_taken_to_load","validationStatus":null,"validationMessage":null},"3":{"slot":3,"fromId":null,"fromName":"Time Taken To Stage","mapTo":"time_taken_to_stage","validationStatus":null,"validationMessage":null},"4":{"slot":4,"fromId":null,"fromName":"Row Count","mapTo":"row_count","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Query Source"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Basic/Advanced Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Advanced"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Authentication","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${oauth}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Data Source","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Data Selection","elements":{},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Data Source Filter","elements":{},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Combine Filters","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":true,"status":{},"columnNames":[]},"10":{"slot":10,"name":"SQL Query","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${sql}"}}}},"visible":true,"status":{},"columnNames":[]},"11":{"slot":11,"name":"Limit","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"100"}}}},"visible":true,"status":{},"columnNames":[]},"12":{"slot":12,"name":"Connection Options","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"connectionOptions"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"parameter"},"2":{"slot":2,"type":"GRID","value":"value"}}}},"visible":true,"status":{},"columnNames":[]},"13":{"slot":13,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${table}"}}}},"visible":true,"status":{},"columnNames":[]},"14":{"slot":14,"name":"Distribution Style","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${dist_style}"}}}},"visible":true,"status":{},"columnNames":[]},"15":{"slot":15,"name":"Distribution Key","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"16":{"slot":16,"name":"Sort Key","elements":{},"visible":true,"status":{},"columnNames":[]},"17":{"slot":17,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${schema}"}}}},"visible":true,"status":{},"columnNames":[]},"18":{"slot":18,"name":"Sort Key Options","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"19":{"slot":19,"name":"S3 Staging Area","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${s3_bucket_name}"}}}},"visible":true,"status":{},"columnNames":[]},"20":{"slot":20,"name":"Data Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot"}}}},"visible":false,"status":{},"columnNames":[]},"21":{"slot":21,"name":"Primary Key","elements":{},"visible":true,"status":{},"columnNames":[]},"96":{"slot":96,"name":"Location","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"s3://<bucket>/<path>"}}}},"visible":false,"status":{},"columnNames":[]},"99":{"slot":99,"name":"Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Standard"}}}},"visible":true,"status":{},"columnNames":[]},"1013":{"slot":1013,"name":"Auto Debug","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${autoDebug}"}}}},"visible":true,"status":{},"columnNames":[]},"1014":{"slot":1014,"name":"Debug Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${autoDebugVerbosity}"}}}},"visible":false,"status":{},"columnNames":[]},"1034":{"slot":1034,"name":"Load Options","elements":{},"visible":true,"status":{},"columnNames":[]},"40000":{"slot":40000,"name":"Encryption","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${encryption_method}"}}}},"visible":true,"status":{},"columnNames":[]},"40001":{"slot":40001,"name":"KMS Key ID","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${kms_key_id}"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31258":{"id":31258,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":166,"y":160,"width":32,"height":32,"inputConnectorIDs":[31268],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[31267],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Print Error"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"msg = \\\n\"\"\"There was an issue whilst retrieving data from the source. \nThe attempted query was: \\n{q}\nError: {e}\n\"\"\".format(q = sql, e = error_message) \nprint(msg)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31259":{"id":31259,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":160,"y":320,"width":32,"height":32,"inputConnectorIDs":[31267],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[31265],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Error"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"error"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Staging Data"},"2":{"slot":2,"type":"STRING","value":"${error_message}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31260":{"id":31260,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":320,"y":320,"width":32,"height":32,"inputConnectorIDs":[31265],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Failure"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31261":{"id":31261,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":320,"y":0,"width":32,"height":32,"inputConnectorIDs":[31263],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31262":{"id":31262,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":4,"y":3,"width":32,"height":32,"inputConnectorIDs":[31266],"outputSuccessConnectorIDs":[31264],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Limit HubSpot Data"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"# hard limit to HubSpot tables that can be set during testing (via `testlimit` connection option)\nif dataLimit:\n  sql += \" LIMIT {}\".format(dataLimit)  \n  context.updateVariable('sql', sql)\n  print(\"New SQL: {}\".format(sql))\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"31263":{"id":31263,"sourceID":31257,"targetID":31261},"31264":{"id":31264,"sourceID":31262,"targetID":31257}},"failureConnectors":{"31268":{"id":31268,"sourceID":31257,"targetID":31258}},"unconditionalConnectors":{"31265":{"id":31265,"sourceID":31259,"targetID":31260},"31266":{"id":31266,"sourceID":31256,"targetID":31262},"31267":{"id":31267,"sourceID":31258,"targetID":31259}},"trueConnectors":{},"falseConnectors":{},"iterationConnectors":{},"noteConnectors":{},"canUndo":false,"undoCommand":"","undoCreated":0,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{},"variables":{"sql":{"definition":{"name":"sql","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"oauth":{"definition":{"name":"oauth","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"table":{"definition":{"name":"table","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"schema":{"definition":{"name":"schema","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"dataset":{"definition":{"name":"dataset","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"project":{"definition":{"name":"project","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"database":{"definition":{"name":"database","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"autoDebug":{"definition":{"name":"autoDebug","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"Off"},"dataLimit":{"definition":{"name":"dataLimit","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"row_count":{"definition":{"name":"row_count","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"},"warehouse":{"definition":{"name":"warehouse","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"dist_style":{"definition":{"name":"dist_style","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"Even"},"kms_key_id":{"definition":{"name":"kms_key_id","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"staging_type":{"definition":{"name":"staging_type","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"Snowflake Managed"},"error_message":{"definition":{"name":"error_message","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"blob_container":{"definition":{"name":"blob_container","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"s3_bucket_name":{"definition":{"name":"s3_bucket_name","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"storage_account":{"definition":{"name":"storage_account","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"encryption_method":{"definition":{"name":"encryption_method","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"None"},"target_table_name":{"definition":{"name":"target_table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"autoDebugVerbosity":{"definition":{"name":"autoDebugVerbosity","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"3"},"cloud_storage_area":{"definition":{"name":"cloud_storage_area","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"time_taken_to_load":{"definition":{"name":"time_taken_to_load","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"},"time_taken_to_stage":{"definition":{"name":"time_taken_to_stage","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"}},"grids":{"connectionOptions":{"definition":{"name":"connectionOptions","scope":"BRANCH","definitions":[{"name":"parameter","type":"TEXT"},{"name":"value","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]}}},"HubSpot - 2 - Wrapper":{"id":-2,"revision":1,"created":1702911151121,"timestamp":1702911151121,"components":{"31343":{"id":31343,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":-240,"y":-80,"width":32,"height":32,"inputConnectorIDs":[31361],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[31363],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log End"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"info"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Iteration Duration"},"2":{"slot":2,"type":"STRING","value":"${iter_duration}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"Iteration End"},"2":{"slot":2,"type":"STRING","value":"${iter_state}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"Iteration Name"},"2":{"slot":2,"type":"STRING","value":"${table_name}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31344":{"id":31344,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":-400,"y":-160,"width":32,"height":32,"inputConnectorIDs":[31360],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Iteration Success"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31345":{"id":31345,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":80,"y":-80,"width":32,"height":32,"inputConnectorIDs":[31365],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Iteration Failure: Error Cleaning Stage Tables"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31346":{"id":31346,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":-624,"y":0,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[31362],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Start","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31347":{"id":31347,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":-400,"y":0,"width":32,"height":32,"inputConnectorIDs":[31362],"outputSuccessConnectorIDs":[31360],"outputFailureConnectorIDs":[31364,31366],"outputUnconditionalConnectorIDs":[31361],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"metadata_stage_table_name","mapTo":"metadata_stage_table_name","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"comp_message","mapTo":"iter_component_message","validationStatus":null,"validationMessage":null},"3":{"slot":3,"fromId":null,"fromName":"comp_name","mapTo":"iter_component_name","validationStatus":null,"validationMessage":null},"4":{"slot":4,"fromId":null,"fromName":"Status","mapTo":"iter_state","validationStatus":null,"validationMessage":null},"5":{"slot":5,"fromId":null,"fromName":"target_table_name","mapTo":"target_table_name","validationStatus":null,"validationMessage":null},"6":{"slot":6,"fromId":null,"fromName":"Duration","mapTo":"iter_duration","validationStatus":null,"validationMessage":null},"7":{"slot":7,"fromId":null,"fromName":"stage_table_name","mapTo":"stage_table_name","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Stage And Load"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot - 3 - Stage and Load"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"staging_type"},"2":{"slot":2,"type":"STRING","value":"${staging_type}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"blob_container"},"2":{"slot":2,"type":"STRING","value":"${blob_container}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"cloud_storage_area"},"2":{"slot":2,"type":"STRING","value":"${cloud_storage_area}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"dist_style"},"2":{"slot":2,"type":"STRING","value":"${dist_style}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"encryption_method"},"2":{"slot":2,"type":"STRING","value":"${encryption_method}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"kms_key_id"},"2":{"slot":2,"type":"STRING","value":"${kms_key_id}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"oauth"},"2":{"slot":2,"type":"STRING","value":"${oauth}"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"s3_bucket_name"},"2":{"slot":2,"type":"STRING","value":"${s3_bucket_name}"}}},"9":{"slot":9,"values":{"1":{"slot":1,"type":"STRING","value":"stage_database"},"2":{"slot":2,"type":"STRING","value":"${stage_database}"}}},"10":{"slot":10,"values":{"1":{"slot":1,"type":"STRING","value":"stage_dataset"},"2":{"slot":2,"type":"STRING","value":"${stage_dataset}"}}},"11":{"slot":11,"values":{"1":{"slot":1,"type":"STRING","value":"stage_prefix"},"2":{"slot":2,"type":"STRING","value":"${stage_prefix}"}}},"12":{"slot":12,"values":{"1":{"slot":1,"type":"STRING","value":"stage_project"},"2":{"slot":2,"type":"STRING","value":"${stage_project}"}}},"13":{"slot":13,"values":{"1":{"slot":1,"type":"STRING","value":"stage_warehouse"},"2":{"slot":2,"type":"STRING","value":"${stage_warehouse}"}}},"14":{"slot":14,"values":{"1":{"slot":1,"type":"STRING","value":"stage_schema"},"2":{"slot":2,"type":"STRING","value":"${stage_schema}"}}},"15":{"slot":15,"values":{"1":{"slot":1,"type":"STRING","value":"target_database"},"2":{"slot":2,"type":"STRING","value":"${target_database}"}}},"16":{"slot":16,"values":{"1":{"slot":1,"type":"STRING","value":"target_dataset"},"2":{"slot":2,"type":"STRING","value":"${target_dataset}"}}},"17":{"slot":17,"values":{"1":{"slot":1,"type":"STRING","value":"target_prefix"},"2":{"slot":2,"type":"STRING","value":"${target_prefix}"}}},"18":{"slot":18,"values":{"1":{"slot":1,"type":"STRING","value":"target_project"},"2":{"slot":2,"type":"STRING","value":"${target_project}"}}},"19":{"slot":19,"values":{"1":{"slot":1,"type":"STRING","value":"target_warehouse"},"2":{"slot":2,"type":"STRING","value":"${target_warehouse}"}}},"20":{"slot":20,"values":{"1":{"slot":1,"type":"STRING","value":"target_schema"},"2":{"slot":2,"type":"STRING","value":"${target_schema}"}}},"21":{"slot":21,"values":{"1":{"slot":1,"type":"STRING","value":"storage_account"},"2":{"slot":2,"type":"STRING","value":"${storage_account}"}}},"22":{"slot":22,"values":{"1":{"slot":1,"type":"STRING","value":"table_name"},"2":{"slot":2,"type":"STRING","value":"${table_name}"}}},"23":{"slot":23,"values":{"1":{"slot":1,"type":"STRING","value":"autoDebug"},"2":{"slot":2,"type":"STRING","value":"${autoDebug}"}}},"24":{"slot":24,"values":{"1":{"slot":1,"type":"STRING","value":"autoDebugVerbosity"},"2":{"slot":2,"type":"STRING","value":"${autoDebugVerbosity}"}}},"25":{"slot":25,"values":{"1":{"slot":1,"type":"STRING","value":"testLimit"},"2":{"slot":2,"type":"STRING","value":"${testLimit}"}}},"26":{"slot":26,"values":{"1":{"slot":1,"type":"STRING","value":"target_platform"},"2":{"slot":2,"type":"STRING","value":"${target_platform}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"connection_options"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"param"},"2":{"slot":2,"type":"GRID","value":"value"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"connection_options"},"2":{"slot":2,"type":"STRING","value":"grid"}}},"2":{"slot":2,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"source_list"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"tableName"},"2":{"slot":2,"type":"GRID","value":"columnName"},"3":{"slot":3,"type":"GRID","value":"isIncremental"},"4":{"slot":4,"type":"GRID","value":"isKey"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"source_list"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot Incremental Load"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Incremental"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31348":{"id":31348,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1291941950,"x":-80,"y":-80,"width":32,"height":32,"inputConnectorIDs":[31363],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[31365],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Drop Stage Tables"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Tables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${metadata_stage_table_name}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Cascade","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Yes"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Ignore Missing","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Yes"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31349":{"id":31349,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":-399,"y":161,"width":32,"height":32,"inputConnectorIDs":[31364],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Iteration Failure"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31350":{"id":31350,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":-240,"y":80,"width":32,"height":32,"inputConnectorIDs":[31366],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Failure"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"error"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Iteration Error"},"2":{"slot":2,"type":"STRING","value":"${iter_component_message ? iter_component_message : \" \"}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"Iteration Error At"},"2":{"slot":2,"type":"STRING","value":"${iter_component_name ? iter_component_name: \" \"}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"31360":{"id":31360,"sourceID":31347,"targetID":31344}},"failureConnectors":{"31364":{"id":31364,"sourceID":31347,"targetID":31349},"31365":{"id":31365,"sourceID":31348,"targetID":31345},"31366":{"id":31366,"sourceID":31347,"targetID":31350}},"unconditionalConnectors":{"31361":{"id":31361,"sourceID":31347,"targetID":31343},"31362":{"id":31362,"sourceID":31346,"targetID":31347},"31363":{"id":31363,"sourceID":31343,"targetID":31348}},"trueConnectors":{},"falseConnectors":{},"iterationConnectors":{},"noteConnectors":{},"canUndo":false,"undoCommand":"","undoCreated":0,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"31342":{"id":31342,"x":-795,"y":-398,"width":952,"height":635,"text":"**Execute ETL Flow for specific source table**\n\nThis job wraps our Stage 3 job (Stage Object). This is done so that we can manage some exported variables from the Stage 3. If we were directly iterating over (Stage Object) we would not be able to handle these exported variables.\n\n- (__Stage And Load__) Passes in the parameters and their values into the Stage - 3 job. Regardless of the result, the __metadata_stage_table_name__ and __stage_table_name__ are exported - this is to ensure these tables are dropped if the process fails before it has time to clean up these staging tables (when a run succeeds, these staging tables will no longer exist).\n\n- (__Drop Stage Tables__) Passes in the exported table names to be deleted. This flow is executed regardless of the result of Stage Object.","colour":"e6e63c"}},"variables":{"oauth":{"definition":{"name":"oauth","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"autoDebug":{"definition":{"name":"autoDebug","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"testLimit":{"definition":{"name":"testLimit","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"dist_style":{"definition":{"name":"dist_style","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"iter_state":{"definition":{"name":"iter_state","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"kms_key_id":{"definition":{"name":"kms_key_id","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"table_name":{"definition":{"name":"table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"log_metrics":{"definition":{"name":"log_metrics","type":"TEXT","scope":"TASKBATCH","description":"TRUE or FALSE (default=TRUE).\nWhether or not to add useful information to the metrics store.\n","visibility":"PUBLIC"},"value":""},"stage_prefix":{"definition":{"name":"stage_prefix","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"stage_schema":{"definition":{"name":"stage_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"staging_type":{"definition":{"name":"staging_type","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"error_message":{"definition":{"name":"error_message","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"iter_duration":{"definition":{"name":"iter_duration","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"0"},"source_schema":{"definition":{"name":"source_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"stage_dataset":{"definition":{"name":"stage_dataset","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"stage_project":{"definition":{"name":"stage_project","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_prefix":{"definition":{"name":"target_prefix","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_schema":{"definition":{"name":"target_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"blob_container":{"definition":{"name":"blob_container","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"s3_bucket_name":{"definition":{"name":"s3_bucket_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"stage_database":{"definition":{"name":"stage_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_dataset":{"definition":{"name":"target_dataset","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_project":{"definition":{"name":"target_project","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"stage_warehouse":{"definition":{"name":"stage_warehouse","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"storage_account":{"definition":{"name":"storage_account","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_database":{"definition":{"name":"target_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_platform":{"definition":{"name":"target_platform","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_table_name":{"definition":{"name":"stage_table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"target_warehouse":{"definition":{"name":"target_warehouse","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"encryption_method":{"definition":{"name":"encryption_method","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_table_name":{"definition":{"name":"target_table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"autoDebugVerbosity":{"definition":{"name":"autoDebugVerbosity","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"cloud_storage_area":{"definition":{"name":"cloud_storage_area","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"iter_component_name":{"definition":{"name":"iter_component_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"limitedSQLStatement":{"definition":{"name":"limitedSQLStatement","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"iter_component_message":{"definition":{"name":"iter_component_message","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"password_manager_entry":{"definition":{"name":"password_manager_entry","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"metadata_stage_table_name":{"definition":{"name":"metadata_stage_table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""}},"grids":{"message":{"definition":{"name":"message","scope":"BRANCH","definitions":[{"name":"key","type":"TEXT"},{"name":"value","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[{"values":["Iteration Duration","${iter_duration}"]},{"values":["Iteration End","${iter_state}"]}]},"source_list":{"definition":{"name":"source_list","scope":"BRANCH","definitions":[{"name":"tableName","type":"TEXT"},{"name":"columnName","type":"TEXT"},{"name":"isIncremental","type":"DECIMAL"},{"name":"isKey","type":"DECIMAL"}],"description":"","visibility":"PUBLIC"},"values":[]},"connection_options":{"definition":{"name":"connection_options","scope":"BRANCH","definitions":[{"name":"param","type":"TEXT"},{"name":"value","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]}}},"HubSpot - 1 - Orchestrator":{"id":-2,"revision":1,"created":1702911151121,"timestamp":1702911151121,"components":{"31274":{"id":31274,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":3,"y":6,"width":32,"height":32,"inputConnectorIDs":[31290],"outputSuccessConnectorIDs":[31284],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Pipeline Start"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"info"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Pipeline Started"},"2":{"slot":2,"type":"STRING","value":" "}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31275":{"id":31275,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":480,"y":0,"width":32,"height":32,"inputConnectorIDs":[31286],"outputSuccessConnectorIDs":[31287],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Update [Environment Default]"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"# Script to replace any [Environment Default] entries with the schema name\nif target_platform.lower() == \"redshift\":\n\n    if stage_schema == \"[Environment Default]\":\n        context.updateVariable(\"stage_schema\", environment_default_schema)\n        print(\n            f\"Updating stage_schema from [Environment Default] to: {environment_default_schema}\"\n        )\n\n    if target_schema == \"[Environment Default]\":\n        context.updateVariable(\"target_schema\", environment_default_schema)\n        print(\n            f\"Updating target_schema from [Environment Default] to: {environment_default_schema}\"\n        )\n\nelif target_platform.lower() == \"snowflake\":\n\n    if target_warehouse == \"[Environment Default]\":\n        context.updateVariable(\"target_warehouse\", environment_port)\n        print(\n            f\"Updating target_warehouse from [Environment Default] to: {environment_port}\"\n        )\n\n    if target_database == \"[Environment Default]\":\n        context.updateVariable(\"target_database\", environment_database)\n        print(\n            f\"Updating target_database from [Environment Default] to: {environment_database}\"\n        )\n\n    if target_schema == \"[Environment Default]\":\n        context.updateVariable(\"target_schema\", environment_default_schema)\n        print(\n            f\"Updating target_schema from [Environment Default] to: {environment_default_schema}\"\n        )\n\n    if stage_warehouse == \"[Environment Default]\":\n        context.updateVariable(\"stage_warehouse\", environment_port)\n        print(\n            f\"Updating stage_warehouse from [Environment Default] to: {environment_port}\"\n        )\n\n    if stage_database == \"[Environment Default]\":\n        context.updateVariable(\"stage_database\", environment_database)\n        print(\n            f\"Updating stage_database from [Environment Default] to: {environment_database}\"\n        )\n\n    if stage_schema == \"[Environment Default]\":\n        context.updateVariable(\"stage_schema\", environment_default_schema)\n        print(\n            f\"Updating stage_schema from [Environment Default] to: {environment_default_schema}\"\n        )\n\nelif target_platform.lower() == \"bigquery\":\n\n    if stage_dataset == \"[Environment Default]\":\n        context.updateVariable(\"stage_dataset\", environment_default_schema)\n        print(\n            f\"Updating stage_dataset from [Environment Default] to: {environment_default_schema}\"\n        )\n\n    if target_dataset == \"[Environment Default]\":\n        context.updateVariable(\"target_dataset\", environment_default_schema)\n        print(\n            f\"Updating target_dataset from [Environment Default] to: {environment_default_schema}\"\n        )\n\n    if stage_project == \"[Environment Default]\":\n        context.updateVariable(\"stage_project\", environment_database)\n        print(\n            f\"Updating stage_project from [Environment Default] to: {environment_default_schema}\"\n        )\n\n    if target_project == \"[Environment Default]\":\n        context.updateVariable(\"target_project\", environment_database)\n        print(\n            f\"Updating target_project from [Environment Default] to: {environment_default_schema}\"\n        )\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31276":{"id":31276,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":800,"y":0,"width":32,"height":32,"inputConnectorIDs":[31288],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Fin"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31277":{"id":31277,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":800,"y":80,"width":32,"height":32,"inputConnectorIDs":[31291],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Failure"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31278":{"id":31278,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":-160,"y":0,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[31290],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31279":{"id":31279,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186960,"x":320,"y":0,"width":32,"height":32,"inputConnectorIDs":[31285],"outputSuccessConnectorIDs":[31286],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Get Unique Table Names"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"tbls = list(set([str(x[0]) for x in context.getGridVariable('source_list')]))\nunique_tbls = [[x] for x in tbls]\n\n# set the Grid Variable \nprint(\"Updated GridVariable 'DistinctTables' with: {}\".format(unique_tbls))\ncontext.updateGridVariable(\"DistinctTables\",unique_tbls)\n\n\ninternal_source_list = [[x[0], x[1], '0', '0'] for x in context.getGridVariable('source_list')]\ncontext.updateGridVariable('internal_source_list', internal_source_list)\n\n\n\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31280":{"id":31280,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"ITERATE","implementationID":-424773870,"x":640,"y":4,"width":32,"height":16,"inputConnectorIDs":[31287],"outputSuccessConnectorIDs":[31288],"outputFailureConnectorIDs":[31291],"outputUnconditionalConnectorIDs":[31289],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"Duration","mapTo":"logging_duration","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"Status","mapTo":"status","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Iterate Through Tables"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Grid Variable","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DistinctTables"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Grid Variable Mapping","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"table_name"},"2":{"slot":2,"type":"STRING","value":"table_name"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Break on Failure","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"No"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Concurrency","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${load_concurrent}"}}}},"visible":true,"status":{},"columnNames":[]},"999":{"slot":999,"name":"Record Values In Task History","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Yes"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[31292],"inputIterationConnectorIDs":[]},"31281":{"id":31281,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":640,"y":4,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"2 - Wrapper"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot - 2 - Wrapper"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"table_name"},"2":{"slot":2,"type":"STRING","value":"${table_name}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"oauth"},"2":{"slot":2,"type":"STRING","value":"${oauth}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"s3_bucket_name"},"2":{"slot":2,"type":"STRING","value":"${s3_bucket_name}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"stage_prefix"},"2":{"slot":2,"type":"STRING","value":"${stage_prefix}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"stage_schema"},"2":{"slot":2,"type":"STRING","value":"${stage_schema}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"target_prefix"},"2":{"slot":2,"type":"STRING","value":"${target_prefix}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"target_schema"},"2":{"slot":2,"type":"STRING","value":"${target_schema}"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"dist_style"},"2":{"slot":2,"type":"STRING","value":"${dist_style}"}}},"9":{"slot":9,"values":{"1":{"slot":1,"type":"STRING","value":"target_platform"},"2":{"slot":2,"type":"STRING","value":"${target_platform}"}}},"10":{"slot":10,"values":{"1":{"slot":1,"type":"STRING","value":"stage_database"},"2":{"slot":2,"type":"STRING","value":"${stage_database}"}}},"11":{"slot":11,"values":{"1":{"slot":1,"type":"STRING","value":"stage_project"},"2":{"slot":2,"type":"STRING","value":"${stage_project}"}}},"12":{"slot":12,"values":{"1":{"slot":1,"type":"STRING","value":"stage_warehouse"},"2":{"slot":2,"type":"STRING","value":"${stage_warehouse}"}}},"13":{"slot":13,"values":{"1":{"slot":1,"type":"STRING","value":"stage_dataset"},"2":{"slot":2,"type":"STRING","value":"${stage_dataset}"}}},"14":{"slot":14,"values":{"1":{"slot":1,"type":"STRING","value":"cloud_storage_area"},"2":{"slot":2,"type":"STRING","value":"${cloud_storage_area}"}}},"15":{"slot":15,"values":{"1":{"slot":1,"type":"STRING","value":"target_database"},"2":{"slot":2,"type":"STRING","value":"${target_database}"}}},"16":{"slot":16,"values":{"1":{"slot":1,"type":"STRING","value":"target_dataset"},"2":{"slot":2,"type":"STRING","value":"${target_dataset}"}}},"17":{"slot":17,"values":{"1":{"slot":1,"type":"STRING","value":"target_project"},"2":{"slot":2,"type":"STRING","value":"${target_project}"}}},"18":{"slot":18,"values":{"1":{"slot":1,"type":"STRING","value":"target_warehouse"},"2":{"slot":2,"type":"STRING","value":"${target_warehouse}"}}},"19":{"slot":19,"values":{"1":{"slot":1,"type":"STRING","value":"blob_container"},"2":{"slot":2,"type":"STRING","value":"${blob_container}"}}},"20":{"slot":20,"values":{"1":{"slot":1,"type":"STRING","value":"encryption_method"},"2":{"slot":2,"type":"STRING","value":"${encryption_method}"}}},"21":{"slot":21,"values":{"1":{"slot":1,"type":"STRING","value":"kms_key_id"},"2":{"slot":2,"type":"STRING","value":"${kms_key_id}"}}},"22":{"slot":22,"values":{"1":{"slot":1,"type":"STRING","value":"staging_type"},"2":{"slot":2,"type":"STRING","value":"${staging_type}"}}},"23":{"slot":23,"values":{"1":{"slot":1,"type":"STRING","value":"autoDebug"},"2":{"slot":2,"type":"STRING","value":"${autoDebug}"}}},"24":{"slot":24,"values":{"1":{"slot":1,"type":"STRING","value":"autoDebugVerbosity"},"2":{"slot":2,"type":"STRING","value":"${autoDebugVerbosity}"}}},"25":{"slot":25,"values":{"1":{"slot":1,"type":"STRING","value":"testLimit"},"2":{"slot":2,"type":"STRING","value":"${testLimit}"}}},"26":{"slot":26,"values":{"1":{"slot":1,"type":"STRING","value":"storage_account"},"2":{"slot":2,"type":"STRING","value":"${storage_account}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"advanced_connection_options"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"parameter"},"2":{"slot":2,"type":"GRID","value":"value"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"connection_options"},"2":{"slot":2,"type":"STRING","value":"grid"}}},"2":{"slot":2,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"internal_source_list"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"tableName"},"2":{"slot":2,"type":"GRID","value":"columnName"},"3":{"slot":3,"type":"GRID","value":"incremental"},"4":{"slot":4,"type":"GRID","value":"key"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"source_list"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot Incremental Load"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Incremental"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[31292]},"31282":{"id":31282,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":156,"y":-7,"width":32,"height":32,"inputConnectorIDs":[31284],"outputSuccessConnectorIDs":[31285],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Magic Variables"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"# this may or may not contain variables which are not valid connection options\n# and are designed to be used elsewhere within the shared job, they need to be removed from the grid \n# to prevent downstream errors \nopts = context.getGridVariable('advanced_connection_options')\n\n# create a new list to store the real connection options \nfiltered = [] \n\n# magic vars is a map: lower case param name => job variable name \nmagic_vars = {\n\t'autodebug': 'autoDebug', # passed into XX Query Components for CData Debugging \n    'verbosity': 'autoDebugVerbosity', # passed into XX Query Components (default = 3)\n    'testlimit': 'testLimit', # HubSpot specific var for limiting the amount of rows returned \n}\n\nif opts: \n  for r in opts:\n    param = r[0]\n    val = r[1]\n    \n    p = param.lower()\n    # check if this is a magic variable \n    # update the underlying job variable if it is \n    if p in magic_vars:\n      context.updateVariable(magic_vars[p], val)\n    # otherwise, its a normal connection option \n    else:\n      filtered.append( [param, val] )\n    \n  context.updateGridVariable('advanced_connection_options', filtered)\n  \n  \n  \n  "}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31283":{"id":31283,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":800,"y":-80,"width":32,"height":32,"inputConnectorIDs":[31289],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Pipeline End"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"info"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Pipeline Duration"},"2":{"slot":2,"type":"STRING","value":"${logging_duration}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"Pipeline End"},"2":{"slot":2,"type":"STRING","value":"${status}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"31284":{"id":31284,"sourceID":31274,"targetID":31282},"31285":{"id":31285,"sourceID":31282,"targetID":31279},"31286":{"id":31286,"sourceID":31279,"targetID":31275},"31287":{"id":31287,"sourceID":31275,"targetID":31280},"31288":{"id":31288,"sourceID":31280,"targetID":31276}},"failureConnectors":{"31291":{"id":31291,"sourceID":31280,"targetID":31277}},"unconditionalConnectors":{"31289":{"id":31289,"sourceID":31280,"targetID":31283},"31290":{"id":31290,"sourceID":31278,"targetID":31274}},"trueConnectors":{},"falseConnectors":{},"iterationConnectors":{"31292":{"id":31292,"sourceID":31280,"targetID":31281}},"noteConnectors":{},"canUndo":false,"undoCommand":"","undoCreated":0,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"31270":{"id":31270,"x":1665,"y":-573,"width":250,"height":58,"text":"** IGNORE **","colour":"d60000"},"31271":{"id":31271,"x":-1389,"y":376,"width":250,"height":58,"text":"** IGNORE **","colour":"d60000"},"31272":{"id":31272,"x":-791,"y":-471,"width":624,"height":388,"text":"**HubSpot Incremental Load** \n\nRoot (Stage 1) Orchestration Job of the Matillion Incremental Load pattern. \nThis job is the entry point of the shared job and is responsible for validating and reformatting some of the supplied data.\nThe pattern aims to synchronize INSERTS and UPDATES in a particular source, to a table within the relevant Cloud Data Warehouse.\n\n**Pattern Overview**\n\nThere are 4 distinct stages, each stage is represented by an orchestration job using the naming convention:\n__**<Source Name> - <Stage Number> - <Stage Name>**__\n\n+ **Stage 1** -  performs input validation and prepares the variables for **Stage 3**\n\n+ **Stage 2** - wrapper for **Stage 3** to ensure that in the case of any errors, relevant staging tables are nicely cleaned up.\n\n+ **Stage 3** - Orchestration job containing the ETL logic for a single source table.\n\n+ **Stage 4** - wrapper orchestration job to Query the underlying source. This is abstracted away from the rest of the logic to make the pattern more reusable.\n\nMore detailed information and specifics relating to each connector can be found within each job.\n\n\n======================================================================================================================================================================================================","colour":"00ce4f"},"31273":{"id":31273,"x":-75,"y":-233,"width":789,"height":310,"text":"** Prepare User Input **\n\n- (__Log Pipeline Start__) Writes to the metrics table.\n\n- (__Magic Variables__) Script that extracts some special variables from connection options. These are used internally to allow us to use 'hidden' variables that will not affect the slots of the shared job. Currently, this is used to switch on CData logging.\n\n- (__Get Unique Table Names__) Script to extract a list of table names from the source. This list is then iterated over, and the same ETL logic is applied to each source. This information is stored in the __distinct_table_names__ grid variable.\n\n- (__Update [Environment Default]__) Script to replace any [Environment Default] values with their actual value.\n\n- (__2 - Wrapper__) Iterate over each of the __distinct_table_names__ and call the lower level ETL process for each table in the source.","colour":"e6e63c"}},"variables":{"oauth":{"definition":{"name":"oauth","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":null},"status":{"definition":{"name":"status","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"autoDebug":{"definition":{"name":"autoDebug","type":"TEXT","scope":"BRANCH","description":"","visibility":"PRIVATE"},"value":"Off"},"testLimit":{"definition":{"name":"testLimit","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"dist_style":{"definition":{"name":"dist_style","type":"TEXT","scope":"BRANCH","description":"The distribution style to be used for both the staged and target data.\nThe options are:\nEVEN: Distribute data across all nodes evenly.\nALL: Distribute a full copy of the data onto each node.\nNote: KEY distribution is not implemented.","visibility":"PUBLIC"},"value":null},"kms_key_id":{"definition":{"name":"kms_key_id","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":null},"table_name":{"definition":{"name":"table_name","type":"TEXT","scope":"BRANCH","description":"","visibility":"PRIVATE"},"value":""},"stage_prefix":{"definition":{"name":"stage_prefix","type":"TEXT","scope":"BRANCH","description":"The prefix to assign to each of the source tables when they are staged. This is user assigned to help avoid any potential clashes with existing tables.\nE.g. If stage_prefix is set to 'stage_' and the table being processed is named 'test_data' then the staged table will be stored as 'stage_test_data'.","visibility":"PUBLIC"},"value":null},"stage_schema":{"definition":{"name":"stage_schema","type":"TEXT","scope":"BRANCH","description":"The schema where the staged data will be stored.","visibility":"PUBLIC"},"value":null},"staging_type":{"definition":{"name":"staging_type","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"Snowflake Managed"},"stage_dataset":{"definition":{"name":"stage_dataset","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":null},"stage_project":{"definition":{"name":"stage_project","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"target_prefix":{"definition":{"name":"target_prefix","type":"TEXT","scope":"BRANCH","description":"The prefix to assign to each of the target tables when they are created/updated. This is user assigned to help avoid any potential clashes with existing tables.\nE.g. If target_prefix is set to 'target_' and the table being processed is named 'test_data' then the staged table will be stored as 'target_test_data'.","visibility":"PUBLIC"},"value":null},"target_schema":{"definition":{"name":"target_schema","type":"TEXT","scope":"BRANCH","description":"The schema where the target data will be stored.","visibility":"PUBLIC"},"value":null},"blob_container":{"definition":{"name":"blob_container","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"s3_bucket_name":{"definition":{"name":"s3_bucket_name","type":"TEXT","scope":"BRANCH","description":"The name of the AWS S3 bucket that will be used for staging the source data.","visibility":"PUBLIC"},"value":null},"stage_database":{"definition":{"name":"stage_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"target_dataset":{"definition":{"name":"target_dataset","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"target_project":{"definition":{"name":"target_project","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"load_concurrent":{"definition":{"name":"load_concurrent","type":"TEXT","scope":"BRANCH","description":"Determines whether the tables being processed should be processed one after the other (Sequential) or at the same time (Concurrent).","visibility":"PUBLIC"},"value":"Sequential"},"stage_warehouse":{"definition":{"name":"stage_warehouse","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"storage_account":{"definition":{"name":"storage_account","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"target_database":{"definition":{"name":"target_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":null},"target_platform":{"definition":{"name":"target_platform","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"Redshift"},"logging_duration":{"definition":{"name":"logging_duration","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"target_warehouse":{"definition":{"name":"target_warehouse","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"encryption_method":{"definition":{"name":"encryption_method","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"None"},"autoDebugVerbosity":{"definition":{"name":"autoDebugVerbosity","type":"TEXT","scope":"BRANCH","description":"","visibility":"PRIVATE"},"value":"3"},"cloud_storage_area":{"definition":{"name":"cloud_storage_area","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":null},"password_manager_entry":{"definition":{"name":"password_manager_entry","type":"TEXT","scope":"BRANCH","description":"The entry name in the password manager to use for the source database connection. This will need to be set up separately from the shared job.\nNote: This applies to the Generic JDBC routine. Different authentication options may be required for the other routines.","visibility":"PRIVATE"},"value":""}},"grids":{"source_list":{"definition":{"name":"source_list","scope":"BRANCH","definitions":[{"name":"table","type":"TEXT"},{"name":"column","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]},"DistinctTables":{"definition":{"name":"DistinctTables","scope":"BRANCH","definitions":[{"name":"table_name","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[{"values":["HubSpotDummyTableIgnoreThis"]}]},"internal_source_list":{"definition":{"name":"internal_source_list","scope":"BRANCH","definitions":[{"name":"tableName","type":"TEXT"},{"name":"columnName","type":"TEXT"},{"name":"incremental","type":"DECIMAL"},{"name":"key","type":"DECIMAL"}],"description":"","visibility":"PRIVATE"},"values":[]},"advanced_connection_options":{"definition":{"name":"advanced_connection_options","scope":"BRANCH","definitions":[{"name":"parameter","type":"TEXT"},{"name":"value","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]}}},"HubSpot - 3 - Stage and Load":{"id":-2,"revision":3,"created":1702919299739,"timestamp":1702919299739,"components":{"31300":{"id":31300,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":2000,"y":0,"width":32,"height":32,"inputConnectorIDs":[31327],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[31336],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Extract Start"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"info"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Extract Start"},"2":{"slot":2,"type":"STRING","value":"${load_strategy}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31301":{"id":31301,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":2160,"y":-160,"width":32,"height":32,"inputConnectorIDs":[31338],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Extract End"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"info"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Extract Stage Duration"},"2":{"slot":2,"type":"STRING","value":"${time_taken_to_stage ? time_taken_to_stage : 0}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"Extract Load Duration"},"2":{"slot":2,"type":"STRING","value":"${time_taken_to_load ? time_taken_to_load : 0}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"Extract Row Count"},"2":{"slot":2,"type":"STRING","value":"${static_row_count}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"Extract End"},"2":{"slot":2,"type":"STRING","value":"${extract_state}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"Extract Duration"},"2":{"slot":2,"type":"STRING","value":"${stage_duration}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31302":{"id":31302,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":2160,"y":160,"width":32,"height":32,"inputConnectorIDs":[31339],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End - Extract Failure"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31303":{"id":31303,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186960,"x":880,"y":0,"width":32,"height":32,"inputConnectorIDs":[31325],"outputSuccessConnectorIDs":[31329],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"Component","mapTo":"comp_name","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"Message","mapTo":"comp_message","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Adjust Source List"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"def truncate(word, length = 12):\n  \"\"\"\n  `truncate()` will convert items to a string and ensure they are the length specified by user \n  Used within stringer function to ensure all columns are the same width \n  \"\"\"\n  if len(word.strip()) <= length:\n    return word\n  return \"{}...\".format( word[:(length-3)] )\n  \ndef stringer(lst, colwidth = 30):\n  \"\"\"\n  `stringer()` is used for printing Grid Variables in a more readable manner.\n  \n  It will handle formatting the grid variable to look more like how it is presented in the product \n  (as a table) \n  \"\"\"\n  msgs = [] \n  for row in lst:\n    msg = \" | \".join(\n    \t[\n        \"{:<{width}}\".format( truncate(str(x), length = colwidth), width = colwidth)\n        for x in row \n        ]\n    )\n    msgs.append(msg)\n    \n  return \"\\n\" + \"\\n\".join(msgs)\n\nsource_list = context.getGridVariable('source_list')\nprint(\"Grid variable 'source_list': {}\".format(stringer(source_list)))\n        \nsource_table_metadata = context.getGridVariable('source_table_metadata')\nprint(\"\\nGrid variable 'source_table_metadata': {}\".format(stringer(source_table_metadata)))\n\ninc_col_options = ['Last Modified Date','Last modified date', 'CreatedAt', 'TimeAddedToList', 'SavedAt']\nprint(\"\\nVariable 'inc_col_options': {}\".format(inc_col_options))\n\n# is_table() checks that a row of a grid variable belongs to the table we \n# are currently iterating over\nis_table = lambda x: x[0] == table_name\n\n# filter source_list to current table and extract list of columns\nsrc = [ x for x in source_list if is_table(x) ] \nrequested_columns = [ x[1] for x in src ]\n\n# extract key and inc columns\n# Note that the true/false values for IsKey are sometimes truncated to three letters, i.e. tru/fal\nkey_columns = [ x[0] for x in source_table_metadata if x[6].lower().startswith('t')]\n\n# loop through ordered list of possible incremental columns, using the first one found\nmetadata_columns = [ x[0] for x in source_table_metadata ]\ninc_columns = []\nfor i in inc_col_options:\n  if i in metadata_columns:\n    inc_columns = [i]\n    break\n\n# add missing columns, preserving order set by user\nall_columns = requested_columns\n\nfor x in key_columns:\n    if x not in all_columns:\n        all_columns.append(x)\n\nfor x in inc_columns:\n    if x not in all_columns:\n        all_columns.append(x)   \n        \n# build new source column list with inc and key col flags\nnew_src = [] \n\nfor c in all_columns:\n    new_src.append(\n        [\n            table_name, \n            c,\n            '1' if c in inc_columns else '0',\n            '1' if c in key_columns else '0'\n  \t\t]\n  \t)\n    \ncontext.updateGridVariable('source_list', new_src)\nprint(\"\\nUpdated grid variable 'source_list': {}\".format(stringer(new_src)))\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31304":{"id":31304,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":0,"y":0,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[31337],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Start","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31305":{"id":31305,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":480,"y":0,"width":32,"height":32,"inputConnectorIDs":[31333],"outputSuccessConnectorIDs":[31326],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"Component","mapTo":"comp_name","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"error_message","mapTo":"comp_message","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Get Source Metadata"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot - 4 - Query Source"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"oauth"},"2":{"slot":2,"type":"STRING","value":"${oauth}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"sql"},"2":{"slot":2,"type":"STRING","value":"${source_metadata_sql}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"table"},"2":{"slot":2,"type":"STRING","value":"${metadata_stage_table_name}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"warehouse"},"2":{"slot":2,"type":"STRING","value":"${stage_warehouse}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"database"},"2":{"slot":2,"type":"STRING","value":"${stage_database}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"schema"},"2":{"slot":2,"type":"STRING","value":"${stage_schema}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"project"},"2":{"slot":2,"type":"STRING","value":"${stage_project}"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"dataset"},"2":{"slot":2,"type":"STRING","value":"${stage_dataset}"}}},"9":{"slot":9,"values":{"1":{"slot":1,"type":"STRING","value":"s3_bucket_name"},"2":{"slot":2,"type":"STRING","value":"${s3_bucket_name}"}}},"10":{"slot":10,"values":{"1":{"slot":1,"type":"STRING","value":"dist_style"},"2":{"slot":2,"type":"STRING","value":"${dist_style}"}}},"11":{"slot":11,"values":{"1":{"slot":1,"type":"STRING","value":"encryption_method"},"2":{"slot":2,"type":"STRING","value":"${encryption_method}"}}},"12":{"slot":12,"values":{"1":{"slot":1,"type":"STRING","value":"kms_key_id"},"2":{"slot":2,"type":"STRING","value":"${kms_key_id}"}}},"13":{"slot":13,"values":{"1":{"slot":1,"type":"STRING","value":"cloud_storage_area"},"2":{"slot":2,"type":"STRING","value":"${cloud_storage_area}"}}},"14":{"slot":14,"values":{"1":{"slot":1,"type":"STRING","value":"blob_container"},"2":{"slot":2,"type":"STRING","value":"${blob_container}"}}},"15":{"slot":15,"values":{"1":{"slot":1,"type":"STRING","value":"storage_account"},"2":{"slot":2,"type":"STRING","value":"${storage_account}"}}},"16":{"slot":16,"values":{"1":{"slot":1,"type":"STRING","value":"target_table_name"},"2":{"slot":2,"type":"STRING","value":"${target_table_name}"}}},"17":{"slot":17,"values":{"1":{"slot":1,"type":"STRING","value":"autoDebug"},"2":{"slot":2,"type":"STRING","value":"${autoDebug}"}}},"18":{"slot":18,"values":{"1":{"slot":1,"type":"STRING","value":"autoDebugVerbosity"},"2":{"slot":2,"type":"STRING","value":"${autoDebugVerbosity}"}}},"19":{"slot":19,"values":{"1":{"slot":1,"type":"STRING","value":"staging_type"},"2":{"slot":2,"type":"STRING","value":"${staging_type}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"connection_options"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"param"},"2":{"slot":2,"type":"GRID","value":"value"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"connectionOptions"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot Incremental Load"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Incremental"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31306":{"id":31306,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":1010676649,"x":160,"y":0,"width":32,"height":32,"inputConnectorIDs":[31337],"outputSuccessConnectorIDs":[31324],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"metadata_stage_table_name","mapTo":"metadata_stage_table_name","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"Component","mapTo":"comp_name","validationStatus":null,"validationMessage":null},"3":{"slot":3,"fromId":null,"fromName":"source_metadata_sql","mapTo":"source_metadata_sql","validationStatus":null,"validationMessage":null},"4":{"slot":4,"fromId":null,"fromName":"get_metadata_sql","mapTo":"get_metadata_sql","validationStatus":null,"validationMessage":null},"5":{"slot":5,"fromId":null,"fromName":"stage_table_name","mapTo":"stage_table_name","validationStatus":null,"validationMessage":null},"6":{"slot":6,"fromId":null,"fromName":"target_table_name","mapTo":"target_table_name","validationStatus":null,"validationMessage":null}},"gridExportMappings":{"1":{"slot":1,"fromGrid":"source_list","toGrid":"source_list","columnMapToFrom":{"isKey":"key_column","isIncremental":"incremental_column","tableName":"table","columnName":"column"},"validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Build Strings"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"stage_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_database}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"stage_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"stage_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_project}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"stage_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"stage_prefix","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_prefix}"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"target_prefix","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_prefix}"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"metadata_stage_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"10":{"slot":10,"name":"stage_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"11":{"slot":11,"name":"target_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"12":{"slot":12,"name":"source_metadata_sql","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"13":{"slot":13,"name":"get_metadata_sql","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"14":{"slot":14,"name":"source_list","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"source_list"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"tableName"},"2":{"slot":2,"type":"GRID","value":"columnName"},"3":{"slot":3,"type":"GRID","value":"isIncremental"},"4":{"slot":4,"type":"GRID","value":"isKey"}}}},"visible":true,"status":{},"columnNames":[]},"15":{"slot":15,"name":"random_suffix","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31307":{"id":31307,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1096987946,"x":640,"y":0,"width":32,"height":32,"inputConnectorIDs":[31326],"outputSuccessConnectorIDs":[31325],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"comp_message","mapTo":"comp_message","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"Component","mapTo":"comp_name","validationStatus":null,"validationMessage":null}},"gridExportMappings":{"1":{"slot":1,"fromGrid":"source_table_metadata","toGrid":"source_table_metadata","columnMapToFrom":{"nullable":"is_nullable","dataTypeName":"data_type_name","isKey":"is_key","length":"length","numericScale":"numeric_scale","ordinal":"ordinal","columnName":"column_name"},"validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Check Source Metadata"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"metadata_stage_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${metadata_stage_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"target_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"get_metadata_sql","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${get_metadata_sql}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"stage_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_database}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"stage_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"stage_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_project}"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"stage_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"source_table_metadata","elements":{},"visible":true,"status":{},"columnNames":[]},"10":{"slot":10,"name":"error_message","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"11":{"slot":11,"name":"comp_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"12":{"slot":12,"name":"comp_message","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31308":{"id":31308,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-231597779,"x":1040,"y":0,"width":32,"height":32,"inputConnectorIDs":[31329],"outputSuccessConnectorIDs":[31321],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"Component","mapTo":"comp_name","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"error_message","mapTo":"comp_message","validationStatus":null,"validationMessage":null}},"gridExportMappings":{"1":{"slot":1,"fromGrid":"required_columns","toGrid":"source_list","columnMapToFrom":{"isKey":"key_column","isIncremental":"incremental_column","tableName":"table","columnName":"column_name"},"validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Check Required Columns"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"target_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"actual_columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"source_table_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"columnName"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"required_columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"source_list"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"tableName"},"2":{"slot":2,"type":"GRID","value":"columnName"},"3":{"slot":3,"type":"GRID","value":"isIncremental"},"4":{"slot":4,"type":"GRID","value":"isKey"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"missing_columns","elements":{},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"error_message","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31309":{"id":31309,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":947479920,"x":1200,"y":0,"width":32,"height":32,"inputConnectorIDs":[31321],"outputSuccessConnectorIDs":[31330],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"incremental_column","mapTo":"incremental_column","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"Component","mapTo":"comp_name","validationStatus":null,"validationMessage":null},"3":{"slot":3,"fromId":null,"fromName":"incremental_column_count","mapTo":"incremental_column_count","validationStatus":null,"validationMessage":null},"4":{"slot":4,"fromId":null,"fromName":"SELECT_STRING","mapTo":"source_column_list","validationStatus":null,"validationMessage":null},"5":{"slot":5,"fromId":null,"fromName":"key_column_count","mapTo":"key_column_count","validationStatus":null,"validationMessage":null}},"gridExportMappings":{"1":{"slot":1,"fromGrid":"key_columns","toGrid":"key_columns","columnMapToFrom":{"columnName":"column_name"},"validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Parameter Parser"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"platform","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_platform}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"incremental_source_list","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"source_list"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"tableName"},"2":{"slot":2,"type":"GRID","value":"columnName"},"3":{"slot":3,"type":"GRID","value":"isIncremental"},"4":{"slot":4,"type":"GRID","value":"isKey"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"source_list","elements":{},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"stage_warehouse","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_warehouse}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"stage_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_database}"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"stage_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"stage_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_project}"}}}},"visible":true,"status":{},"columnNames":[]},"10":{"slot":10,"name":"stage_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"11":{"slot":11,"name":"stage_prefix","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_prefix}"}}}},"visible":true,"status":{},"columnNames":[]},"12":{"slot":12,"name":"target_warehouse","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_warehouse}"}}}},"visible":true,"status":{},"columnNames":[]},"13":{"slot":13,"name":"target_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"14":{"slot":14,"name":"target_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"15":{"slot":15,"name":"target_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_project}"}}}},"visible":true,"status":{},"columnNames":[]},"16":{"slot":16,"name":"target_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"17":{"slot":17,"name":"target_prefix","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_prefix}"}}}},"visible":true,"status":{},"columnNames":[]},"18":{"slot":18,"name":"maximum_incremental_columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"DECIMAL","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"19":{"slot":19,"name":"source_enclosing_char_e","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"\""}}}},"visible":true,"status":{},"columnNames":[]},"20":{"slot":20,"name":"source_enclosing_char_s","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"\""}}}},"visible":true,"status":{},"columnNames":[]},"21":{"slot":21,"name":"metadata_stage_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${metadata_stage_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"22":{"slot":22,"name":"stage_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"23":{"slot":23,"name":"target_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"24":{"slot":24,"name":"column_count","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"DECIMAL","value":"0"}}}},"visible":true,"status":{},"columnNames":[]},"25":{"slot":25,"name":"key_column_count","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"DECIMAL","value":"0"}}}},"visible":true,"status":{},"columnNames":[]},"26":{"slot":26,"name":"key_columns","elements":{},"visible":true,"status":{},"columnNames":[]},"27":{"slot":27,"name":"incremental_column_count","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"DECIMAL","value":"0"}}}},"visible":true,"status":{},"columnNames":[]},"28":{"slot":28,"name":"incremental_column","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"29":{"slot":29,"name":"SELECT_STRING","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"30":{"slot":30,"name":"stage_table_metadata_sql","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31310":{"id":31310,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":1440,"y":0,"width":32,"height":32,"inputConnectorIDs":[31330],"outputSuccessConnectorIDs":[31328],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Build stage_single_sql"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"# set base stage_single_sql\nstage_single_sql = (f'SELECT {source_column_list} \\n' + \n                    f'FROM \"{table_name}\" \\n'\n                    f'LIMIT 1'\n)\n\ncontext.updateVariable('stage_single_sql', stage_single_sql)\nprint(f\"Updated Variable 'stage_single_sql' with {stage_single_sql}\")\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31311":{"id":31311,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":1600,"y":0,"width":32,"height":32,"inputConnectorIDs":[31328],"outputSuccessConnectorIDs":[31331],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"Component","mapTo":"comp_name","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"error_message","mapTo":"comp_message","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Stage Single Row"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot - 4 - Query Source"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"oauth"},"2":{"slot":2,"type":"STRING","value":"${oauth}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"sql"},"2":{"slot":2,"type":"STRING","value":"${stage_single_sql}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"table"},"2":{"slot":2,"type":"STRING","value":"${stage_table_name}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"warehouse"},"2":{"slot":2,"type":"STRING","value":"${stage_warehouse}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"database"},"2":{"slot":2,"type":"STRING","value":"${stage_database}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"schema"},"2":{"slot":2,"type":"STRING","value":"${stage_schema}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"project"},"2":{"slot":2,"type":"STRING","value":"${stage_project}"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"dataset"},"2":{"slot":2,"type":"STRING","value":"${stage_dataset}"}}},"9":{"slot":9,"values":{"1":{"slot":1,"type":"STRING","value":"s3_bucket_name"},"2":{"slot":2,"type":"STRING","value":"${s3_bucket_name}"}}},"10":{"slot":10,"values":{"1":{"slot":1,"type":"STRING","value":"dist_style"},"2":{"slot":2,"type":"STRING","value":"${dist_style}"}}},"11":{"slot":11,"values":{"1":{"slot":1,"type":"STRING","value":"encryption_method"},"2":{"slot":2,"type":"STRING","value":"${encryption_method}"}}},"12":{"slot":12,"values":{"1":{"slot":1,"type":"STRING","value":"kms_key_id"},"2":{"slot":2,"type":"STRING","value":"${kms_key_id}"}}},"13":{"slot":13,"values":{"1":{"slot":1,"type":"STRING","value":"cloud_storage_area"},"2":{"slot":2,"type":"STRING","value":"${cloud_storage_area}"}}},"14":{"slot":14,"values":{"1":{"slot":1,"type":"STRING","value":"blob_container"},"2":{"slot":2,"type":"STRING","value":"${blob_container}"}}},"15":{"slot":15,"values":{"1":{"slot":1,"type":"STRING","value":"storage_account"},"2":{"slot":2,"type":"STRING","value":"${storage_account}"}}},"16":{"slot":16,"values":{"1":{"slot":1,"type":"STRING","value":"target_table_name"},"2":{"slot":2,"type":"STRING","value":"${target_table_name}"}}},"17":{"slot":17,"values":{"1":{"slot":1,"type":"STRING","value":"autoDebug"},"2":{"slot":2,"type":"STRING","value":"${autoDebug}"}}},"18":{"slot":18,"values":{"1":{"slot":1,"type":"STRING","value":"autoDebugVerbosity"},"2":{"slot":2,"type":"STRING","value":"${autoDebugVerbosity}"}}},"19":{"slot":19,"values":{"1":{"slot":1,"type":"STRING","value":"staging_type"},"2":{"slot":2,"type":"STRING","value":"${staging_type}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"connection_options"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"param"},"2":{"slot":2,"type":"GRID","value":"value"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"connectionOptions"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot Incremental Load"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Incremental"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31312":{"id":31312,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":332733136,"x":1760,"y":0,"width":32,"height":32,"inputConnectorIDs":[31331],"outputSuccessConnectorIDs":[31327],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"load_strategy","mapTo":"load_strategy","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"comp_message","mapTo":"comp_message","validationStatus":null,"validationMessage":null},"3":{"slot":3,"fromId":null,"fromName":"comp_name","mapTo":"comp_name","validationStatus":null,"validationMessage":null},"4":{"slot":4,"fromId":null,"fromName":"stage_sql","mapTo":"stage_sql","validationStatus":null,"validationMessage":null},"5":{"slot":5,"fromId":null,"fromName":"inc_col_type","mapTo":"inc_col_type","validationStatus":null,"validationMessage":null},"6":{"slot":6,"fromId":null,"fromName":"max_value","mapTo":"max_value","validationStatus":null,"validationMessage":null},"7":{"slot":7,"fromId":null,"fromName":"incremental_column","mapTo":"incremental_column","validationStatus":null,"validationMessage":null}},"gridExportMappings":{"1":{"slot":1,"fromGrid":"stage_table_metadata","toGrid":"stage_table_metadata","columnMapToFrom":{"databaseType":"database_type","size":"size","type":"type","precision":"precision","columnName":"column_name"},"validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Decide Load Strategy"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"source_column_list","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${source_column_list}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"key_column_count","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${key_column_count}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"incremental_column_count","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${incremental_column_count}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"incremental_column","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${incremental_column}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"skip_inc_col_check","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"N"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"timestamp_precision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${timestamp_precision}"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"safely_cast_datetime_to_date","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"N"}}}},"visible":true,"status":{},"columnNames":[]},"10":{"slot":10,"name":"encloseTableString","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"TRUE"}}}},"visible":true,"status":{},"columnNames":[]},"11":{"slot":11,"name":"source_enclosing_char_s","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"\""}}}},"visible":true,"status":{},"columnNames":[]},"12":{"slot":12,"name":"source_enclosing_char_e","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"\""}}}},"visible":true,"status":{},"columnNames":[]},"13":{"slot":13,"name":"target_platform","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_platform}"}}}},"visible":true,"status":{},"columnNames":[]},"14":{"slot":14,"name":"stage_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_database}"}}}},"visible":true,"status":{},"columnNames":[]},"15":{"slot":15,"name":"stage_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"16":{"slot":16,"name":"stage_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_project}"}}}},"visible":true,"status":{},"columnNames":[]},"17":{"slot":17,"name":"stage_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"18":{"slot":18,"name":"stage_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"19":{"slot":19,"name":"target_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"20":{"slot":20,"name":"target_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"21":{"slot":21,"name":"target_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_project}"}}}},"visible":true,"status":{},"columnNames":[]},"22":{"slot":22,"name":"target_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"23":{"slot":23,"name":"target_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"24":{"slot":24,"name":"fix_target_data_types","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Yes"}}}},"visible":true,"status":{},"columnNames":[]},"25":{"slot":25,"name":"load_strategy","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"26":{"slot":26,"name":"max_value","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"27":{"slot":27,"name":"stage_sql","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"28":{"slot":28,"name":"inc_col_type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"29":{"slot":29,"name":"stage_table_metadata","elements":{},"visible":true,"status":{},"columnNames":[]},"30":{"slot":30,"name":"messages","elements":{},"visible":true,"status":{},"columnNames":[]},"31":{"slot":31,"name":"use_incremental_column","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Y"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31313":{"id":31313,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":2160,"y":0,"width":32,"height":32,"inputConnectorIDs":[31336],"outputSuccessConnectorIDs":[31323],"outputFailureConnectorIDs":[31339],"outputUnconditionalConnectorIDs":[31338],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"row_count","mapTo":"static_row_count","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"Component","mapTo":"comp_name","validationStatus":null,"validationMessage":null},"3":{"slot":3,"fromId":null,"fromName":"error_message","mapTo":"comp_message","validationStatus":null,"validationMessage":null},"4":{"slot":4,"fromId":null,"fromName":"Status","mapTo":"extract_state","validationStatus":null,"validationMessage":null},"5":{"slot":5,"fromId":null,"fromName":"Duration","mapTo":"stage_duration","validationStatus":null,"validationMessage":null},"6":{"slot":6,"fromId":null,"fromName":"time_taken_to_load","mapTo":"time_taken_to_load","validationStatus":null,"validationMessage":null},"7":{"slot":7,"fromId":null,"fromName":"time_taken_to_stage","mapTo":"time_taken_to_stage","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Stage Data"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot - 4 - Query Source"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"oauth"},"2":{"slot":2,"type":"STRING","value":"${oauth}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"sql"},"2":{"slot":2,"type":"STRING","value":"${stage_sql}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"table"},"2":{"slot":2,"type":"STRING","value":"${stage_table_name}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"warehouse"},"2":{"slot":2,"type":"STRING","value":"${stage_warehouse}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"database"},"2":{"slot":2,"type":"STRING","value":"${stage_database}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"schema"},"2":{"slot":2,"type":"STRING","value":"${stage_schema}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"project"},"2":{"slot":2,"type":"STRING","value":"${stage_project}"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"dataset"},"2":{"slot":2,"type":"STRING","value":"${stage_dataset}"}}},"9":{"slot":9,"values":{"1":{"slot":1,"type":"STRING","value":"s3_bucket_name"},"2":{"slot":2,"type":"STRING","value":"${s3_bucket_name}"}}},"10":{"slot":10,"values":{"1":{"slot":1,"type":"STRING","value":"dist_style"},"2":{"slot":2,"type":"STRING","value":"${dist_style}"}}},"11":{"slot":11,"values":{"1":{"slot":1,"type":"STRING","value":"encryption_method"},"2":{"slot":2,"type":"STRING","value":"${encryption_method}"}}},"12":{"slot":12,"values":{"1":{"slot":1,"type":"STRING","value":"kms_key_id"},"2":{"slot":2,"type":"STRING","value":"${kms_key_id}"}}},"13":{"slot":13,"values":{"1":{"slot":1,"type":"STRING","value":"cloud_storage_area"},"2":{"slot":2,"type":"STRING","value":"${cloud_storage_area}"}}},"14":{"slot":14,"values":{"1":{"slot":1,"type":"STRING","value":"blob_container"},"2":{"slot":2,"type":"STRING","value":"${blob_container}"}}},"15":{"slot":15,"values":{"1":{"slot":1,"type":"STRING","value":"storage_account"},"2":{"slot":2,"type":"STRING","value":"${storage_account}"}}},"16":{"slot":16,"values":{"1":{"slot":1,"type":"STRING","value":"target_table_name"},"2":{"slot":2,"type":"STRING","value":"${target_table_name}"}}},"17":{"slot":17,"values":{"1":{"slot":1,"type":"STRING","value":"autoDebug"},"2":{"slot":2,"type":"STRING","value":"${autoDebug}"}}},"18":{"slot":18,"values":{"1":{"slot":1,"type":"STRING","value":"autoDebugVerbosity"},"2":{"slot":2,"type":"STRING","value":"${autoDebugVerbosity}"}}},"19":{"slot":19,"values":{"1":{"slot":1,"type":"STRING","value":"staging_type"},"2":{"slot":2,"type":"STRING","value":"${staging_type}"}}},"20":{"slot":20,"values":{"1":{"slot":1,"type":"STRING","value":"dataLimit"},"2":{"slot":2,"type":"STRING","value":"${testLimit}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"connection_options"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"param"},"2":{"slot":2,"type":"GRID","value":"value"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"connectionOptions"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot Incremental Load"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Incremental"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31314":{"id":31314,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-313802845,"x":2480,"y":0,"width":32,"height":32,"inputConnectorIDs":[31335],"outputSuccessConnectorIDs":[31322],"outputFailureConnectorIDs":[31340],"outputUnconditionalConnectorIDs":[31332],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"Status","mapTo":"load_state","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"comp_message","mapTo":"comp_message","validationStatus":null,"validationMessage":null},"3":{"slot":3,"fromId":null,"fromName":"comp_name","mapTo":"comp_name","validationStatus":null,"validationMessage":null},"4":{"slot":4,"fromId":null,"fromName":"logged_rows_deleted","mapTo":"load_deleted","validationStatus":null,"validationMessage":null},"5":{"slot":5,"fromId":null,"fromName":"logged_rows_inserted","mapTo":"load_inserted","validationStatus":null,"validationMessage":null},"6":{"slot":6,"fromId":null,"fromName":"logged_rows_update","mapTo":"load_updated","validationStatus":null,"validationMessage":null},"7":{"slot":7,"fromId":null,"fromName":"Duration","mapTo":"load_duration","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Load Target"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"platform","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_platform}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"stage_warehouse","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_warehouse}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"stage_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_database}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"stage_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"stage_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_project}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"stage_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"stage_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"target_warehouse","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_warehouse}"}}}},"visible":true,"status":{},"columnNames":[]},"10":{"slot":10,"name":"target_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"11":{"slot":11,"name":"target_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"12":{"slot":12,"name":"target_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_project}"}}}},"visible":true,"status":{},"columnNames":[]},"13":{"slot":13,"name":"target_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"14":{"slot":14,"name":"target_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"15":{"slot":15,"name":"load_strategy","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${load_strategy}"}}}},"visible":true,"status":{},"columnNames":[]},"16":{"slot":16,"name":"dist_style","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${dist_style}"}}}},"visible":true,"status":{},"columnNames":[]},"17":{"slot":17,"name":"columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"stage_table_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"columnName"}}}},"visible":true,"status":{},"columnNames":[]},"18":{"slot":18,"name":"key_columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"key_columns"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"columnName"}}}},"visible":true,"status":{},"columnNames":[]},"19":{"slot":19,"name":"incremental_column","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${incremental_column}"}}}},"visible":true,"status":{},"columnNames":[]},"20":{"slot":20,"name":"join_statement","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"21":{"slot":21,"name":"log_metrics","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"TRUE"}}}},"visible":true,"status":{},"columnNames":[]},"22":{"slot":22,"name":"static_row_count","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${static_row_count}"}}}},"visible":true,"status":{},"columnNames":[]},"23":{"slot":23,"name":"logged_rows_inserted","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"DECIMAL","value":"0"}}}},"visible":true,"status":{},"columnNames":[]},"24":{"slot":24,"name":"logged_rows_update","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"DECIMAL","value":"0"}}}},"visible":true,"status":{},"columnNames":[]},"25":{"slot":25,"name":"logged_rows_deleted","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"DECIMAL","value":"0"}}}},"visible":true,"status":{},"columnNames":[]},"26":{"slot":26,"name":"sub_RangeStartDate","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"27":{"slot":27,"name":"index_type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Clustered Column Store"}}}},"visible":true,"status":{},"columnNames":[]},"28":{"slot":28,"name":"comp_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"29":{"slot":29,"name":"comp_message","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31315":{"id":31315,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":2640,"y":0,"width":32,"height":32,"inputConnectorIDs":[31322],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Success"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31316":{"id":31316,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":2320,"y":0,"width":32,"height":32,"inputConnectorIDs":[31323],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[31335],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Load Start"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"info"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Load Start"},"2":{"slot":2,"type":"STRING","value":"${load_strategy}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31317":{"id":31317,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":2480,"y":-160,"width":32,"height":32,"inputConnectorIDs":[31332],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Load End"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"info"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Load Inserted Row Count"},"2":{"slot":2,"type":"STRING","value":"${load_inserted}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"Load Updated Row Count"},"2":{"slot":2,"type":"STRING","value":"${load_updated}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"Load Deleted Row Count"},"2":{"slot":2,"type":"STRING","value":"${load_deleted}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"Load Duration"},"2":{"slot":2,"type":"STRING","value":"${load_duration}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"Load End"},"2":{"slot":2,"type":"STRING","value":"${load_state}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31318":{"id":31318,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":2640,"y":160,"width":32,"height":32,"inputConnectorIDs":[31334],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End - Load Failure"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31319":{"id":31319,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":320,"y":0,"width":32,"height":32,"inputConnectorIDs":[31324],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[31333],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Start"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"info"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Iteration Started"},"2":{"slot":2,"type":"STRING","value":"${table_name}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"31320":{"id":31320,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":2480,"y":160,"width":32,"height":32,"inputConnectorIDs":[31340],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[31334],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Load Error"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"error"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Load Error"},"2":{"slot":2,"type":"STRING","value":"${comp_message}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"Load Error At"},"2":{"slot":2,"type":"STRING","value":"${comp_name}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"31321":{"id":31321,"sourceID":31308,"targetID":31309},"31322":{"id":31322,"sourceID":31314,"targetID":31315},"31323":{"id":31323,"sourceID":31313,"targetID":31316},"31324":{"id":31324,"sourceID":31306,"targetID":31319},"31325":{"id":31325,"sourceID":31307,"targetID":31303},"31326":{"id":31326,"sourceID":31305,"targetID":31307},"31327":{"id":31327,"sourceID":31312,"targetID":31300},"31328":{"id":31328,"sourceID":31310,"targetID":31311},"31329":{"id":31329,"sourceID":31303,"targetID":31308},"31330":{"id":31330,"sourceID":31309,"targetID":31310},"31331":{"id":31331,"sourceID":31311,"targetID":31312}},"failureConnectors":{"31339":{"id":31339,"sourceID":31313,"targetID":31302},"31340":{"id":31340,"sourceID":31314,"targetID":31320}},"unconditionalConnectors":{"31332":{"id":31332,"sourceID":31314,"targetID":31317},"31333":{"id":31333,"sourceID":31319,"targetID":31305},"31334":{"id":31334,"sourceID":31320,"targetID":31318},"31335":{"id":31335,"sourceID":31316,"targetID":31314},"31336":{"id":31336,"sourceID":31300,"targetID":31313},"31337":{"id":31337,"sourceID":31304,"targetID":31306},"31338":{"id":31338,"sourceID":31313,"targetID":31301}},"trueConnectors":{},"falseConnectors":{},"iterationConnectors":{},"noteConnectors":{},"canUndo":true,"undoCommand":"Set Parameter","undoCreated":1702916035157,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"31294":{"id":31294,"x":3920,"y":-1342,"width":250,"height":58,"text":"**IGNORE**","colour":"d60000"},"31295":{"id":31295,"x":-2114,"y":644,"width":250,"height":58,"text":"**IGNORE**","colour":"d60000"},"31296":{"id":31296,"x":85,"y":-241,"width":630,"height":318,"text":"** Source Validation **\n\n- (__Build Strings__) Generate generic job variables (stage/target table names etc. - see Export tab) \n\n- (__Get Source Metadata__) Query CData __sys_tablecolumns__ metadata table to get information about the source and store in CDW \n\n- (__Check Source Metadata__) Query table created above, ensure it is valid, export the metadata to a grid (see Grid Export tab)","colour":"e6e63c"},"31297":{"id":31297,"x":804,"y":-237,"width":472,"height":311,"text":"**Input/Target Validation**\n\n- (__Check Required Columns__) Contains logic to validate that the columns which have been requested still exist in the source\n\n- (__Adjust Source List__) Modifies the current __source_list__ grid variable. This will set the correct key and incremental columns (where applicable). \nNB: For HubSpot, the __'Last Modified Date'__,  __'Last modified date'__,  __'CreatedAt'__  column is used as an incremental column when present.\n\n- (__Parameter Parser__) Contains logic to generate some useful Grid/Job variables (e.g key_columns Grid, SELECT list. See Export/Grid Export tabs)","colour":"e6e63c"},"31298":{"id":31298,"x":1366,"y":-235,"width":466,"height":310,"text":"**Decide Load Strategy**\n\n- (__Build stage_single_sql__) Generates and sets stage_single_sql variable which is used to query 1 row from the source. This logic is in its own Python script so that we can log the query.\n\n- (__Stage Single Row__) Executes the __stage_single_sql__ query. This creates a table which will be identical to the stage table for the ETL process, but with only 1 row - the metadata of this table is used within Decide Load Strategy.\n\n- (__Decide Load Strategy__) Executes packaged up logic to determine the type of load this job is doing (UPDATE/APPEND INSERT/TRUNCATE INSERT). \nThis exports the __stage_sql__ variable which will be used to stage the dataset.\nThis exports the __load_strategy__ variable which will be used within Load Target","colour":"e6e63c"},"31299":{"id":31299,"x":1925,"y":-314,"width":631,"height":550,"text":"** Load Data **\n\n- (__Log Extract Start__) Writes metrics to indicate the Extraction phase has started\n\n- (__Stage Data__) Executes __stage_sql__ query and stores the data in the staging table \n\n- (__Load Target__) Executes logic to move the Stage Table Data into the Target Table using the desired __Load Strategy__","colour":"e6e63c"}},"variables":{"oauth":{"definition":{"name":"oauth","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"autoDebug":{"definition":{"name":"autoDebug","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"comp_name":{"definition":{"name":"comp_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"max_value":{"definition":{"name":"max_value","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"stage_sql":{"definition":{"name":"stage_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"testLimit":{"definition":{"name":"testLimit","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"dist_style":{"definition":{"name":"dist_style","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"kms_key_id":{"definition":{"name":"kms_key_id","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"load_state":{"definition":{"name":"load_state","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"table_name":{"definition":{"name":"table_name","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"log_metrics":{"definition":{"name":"log_metrics","type":"TEXT","scope":"BRANCH","description":"TRUE or FALSE (default=TRUE).\nWhether or not to add useful information to the metrics store.\n","visibility":"PUBLIC"},"value":""},"comp_message":{"definition":{"name":"comp_message","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"inc_col_type":{"definition":{"name":"inc_col_type","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"load_deleted":{"definition":{"name":"load_deleted","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"0"},"load_updated":{"definition":{"name":"load_updated","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"0"},"stage_prefix":{"definition":{"name":"stage_prefix","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_schema":{"definition":{"name":"stage_schema","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"staging_type":{"definition":{"name":"staging_type","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"extract_state":{"definition":{"name":"extract_state","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"load_duration":{"definition":{"name":"load_duration","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"load_inserted":{"definition":{"name":"load_inserted","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"0"},"load_strategy":{"definition":{"name":"load_strategy","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"TRUNCATE INSERT"},"stage_dataset":{"definition":{"name":"stage_dataset","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_project":{"definition":{"name":"stage_project","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_prefix":{"definition":{"name":"target_prefix","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_schema":{"definition":{"name":"target_schema","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"blob_container":{"definition":{"name":"blob_container","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"s3_bucket_name":{"definition":{"name":"s3_bucket_name","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_database":{"definition":{"name":"stage_database","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_duration":{"definition":{"name":"stage_duration","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"},"target_dataset":{"definition":{"name":"target_dataset","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_project":{"definition":{"name":"target_project","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_warehouse":{"definition":{"name":"stage_warehouse","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"storage_account":{"definition":{"name":"storage_account","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_database":{"definition":{"name":"target_database","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_platform":{"definition":{"name":"target_platform","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"extract_duration":{"definition":{"name":"extract_duration","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"get_metadata_sql":{"definition":{"name":"get_metadata_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"key_column_count":{"definition":{"name":"key_column_count","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"stage_single_sql":{"definition":{"name":"stage_single_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"stage_table_name":{"definition":{"name":"stage_table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"static_row_count":{"definition":{"name":"static_row_count","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"target_warehouse":{"definition":{"name":"target_warehouse","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"encryption_method":{"definition":{"name":"encryption_method","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_table_name":{"definition":{"name":"target_table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"target_table_name"},"autoDebugVerbosity":{"definition":{"name":"autoDebugVerbosity","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"cloud_storage_area":{"definition":{"name":"cloud_storage_area","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"incremental_column":{"definition":{"name":"incremental_column","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"source_column_list":{"definition":{"name":"source_column_list","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"time_taken_to_load":{"definition":{"name":"time_taken_to_load","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"},"limitedSQLStatement":{"definition":{"name":"limitedSQLStatement","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"source_metadata_sql":{"definition":{"name":"source_metadata_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"time_taken_to_stage":{"definition":{"name":"time_taken_to_stage","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"},"timestamp_precision":{"definition":{"name":"timestamp_precision","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"microseconds"},"incremental_column_count":{"definition":{"name":"incremental_column_count","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"0"},"metadata_stage_table_name":{"definition":{"name":"metadata_stage_table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""}},"grids":{"key_columns":{"definition":{"name":"key_columns","scope":"BRANCH","definitions":[{"name":"columnName","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[]},"source_list":{"definition":{"name":"source_list","scope":"BRANCH","definitions":[{"name":"tableName","type":"TEXT"},{"name":"columnName","type":"TEXT"},{"name":"isIncremental","type":"DECIMAL"},{"name":"isKey","type":"DECIMAL"}],"description":"","visibility":"PUBLIC"},"values":[]},"connection_options":{"definition":{"name":"connection_options","scope":"BRANCH","definitions":[{"name":"param","type":"TEXT"},{"name":"value","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]},"stage_table_metadata":{"definition":{"name":"stage_table_metadata","scope":"BRANCH","definitions":[{"name":"columnName","type":"TEXT"},{"name":"type","type":"TEXT"},{"name":"size","type":"DECIMAL"},{"name":"precision","type":"DECIMAL"},{"name":"databaseType","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[]},"source_table_metadata":{"definition":{"name":"source_table_metadata","scope":"BRANCH","definitions":[{"name":"columnName","type":"TEXT"},{"name":"dataTypeName","type":"TEXT"},{"name":"length","type":"TEXT"},{"name":"numericScale","type":"TEXT"},{"name":"nullable","type":"TEXT"},{"name":"ordinal","type":"TEXT"},{"name":"isKey","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[]}}}},"transformationJobs":{}}],"version":"Built","environment":"redshift"}