{"objects":[{"metadata":{"identifier":{"packageName":"Matillion.Incremental","name":"HubSpot Incremental Load","revision":1,"type":"DYNAMIC"},"rootJobReference":{"name":"HubSpot - 1 - Orchestrator","type":"ORCHESTRATION","parameterMetadata":[{"slot":2,"variableName":"oauth","variableType":"SCALAR","displayName":"OAuth","description":"The name of the OAuth entry to be used from the Matillion OAuth Manager to connect to the source. These must be set up in advance, using Project --> Mangage OAuth.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":3,"variableName":"source_list","variableType":"GRID","displayName":"Tables and Columns","description":"Contains the list of tables and columns (and in some cases an incremental_column to specify which column the load should be incremented on [accepts values of 0 or 1]) to be processed. ","defaultValue":[{"values":{"1":""}}],"defaultValueType":null,"required":true},{"slot":4,"variableName":"advanced_connection_options","variableType":"GRID","displayName":"Connection Options","description":"A list of values and parameters. Parameters and their allowed values are database/driver specific. Referring to the data model will provide insight of what you could provide here.\n<br>\nThey are usually not required as sensible defaults are assumed.","defaultValue":[{"values":{"1":""}}],"defaultValueType":null,"required":false},{"slot":5,"variableName":"cloud_storage_area","variableType":"SCALAR","displayName":"Cloud Storage Area","description":"The URL and path of the target Google Storage bucket to be used for staging the queried data throughout the job.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":6,"variableName":"load_concurrent","variableType":"SCALAR","displayName":"Load Type","description":"Sequential - Iterations are done in sequence, waiting for each to complete before starting the next. \nThis is the default.\n<br>\nConcurrent - Iterations are run concurrently. This requires all \"Variables to Iterate\" to be defined as\n<a href=\"https://redshiftsupport.matillion.com/customer/portal/articles/2037630?b_id=8915#header1\" target=\"_blank\"><i>Copied</i> variables</a>, so that each iteration gets its own copy of the variable isolated from the same\nvariable being used by other concurrent executions.\n<span class=\"env rs sf\"><br><b>Note:</b> The maximum concurrency is limited by the number of available threads (2x the number of virtual cpus on your cloud instance).</span>","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":7,"variableName":"stage_project","variableType":"SCALAR","displayName":"Stage Project","description":"The project name where the staging data will be stored.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":8,"variableName":"stage_dataset","variableType":"SCALAR","displayName":"Stage Dataset","description":"The dataset name where the staging data will be stored.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":9,"variableName":"stage_prefix","variableType":"SCALAR","displayName":"Stage Prefix","description":"A prefix value that will be added to the start of the stage table names.\n<br>\ne.g. If a Stage Prefix of 'stage_' is specified and the table being processed is named 'test_data' then the target table will be named 'stage_test_data'.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":10,"variableName":"target_project","variableType":"SCALAR","displayName":"Target Project","description":"The project name where the target data will be stored.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":11,"variableName":"target_dataset","variableType":"SCALAR","displayName":"Target Dataset","description":"The dataset name where the target data will be stored.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":12,"variableName":"target_prefix","variableType":"SCALAR","displayName":"Target Prefix","description":"A prefix value that will be added to the start of the target table names.\n<br>\ne.g. If a Target Prefix of 'target_' is specified and the table being processed is named 'test_data' then the target table will be named 'target_test_data'.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":false}]},"orchestrationJobs":["HubSpot - 1 - Orchestrator","HubSpot - 2 - Wrapper","HubSpot - 3 - Stage and Load","HubSpot - 4 - Query Source"],"transformationJobs":[],"imageMetadata":{"componentIcon":"iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAADMUlEQVR42t1XXUgUURjdfqnIyCio3sR8KIrCh+ilngrqwZdi2b2za4nVYvRgFo1z70w1RokFERL0UhCRYEUgZBYRFm710INP/aEUYVKCZBaiGe76dWYd1zuza+6uk0UDHzPsne+ec88933dnfb5/8SK/f87fAdaDARJKFwkWR0TJUIpymkdT8sksW5BdkggUAzSGICk6slEDoEtJVx7auUPEmZoFgeBpF7gdyqaM5+BKfUq+rmzOMJmpaQkYyvbMF8GephIIVmSWfNy/Egm9aUgMWN744wqMmSe8BmBXocYjJJ9BfEhOxNlFikTmTekBzp7YOfGsPJB+Qv8yeOC+tKIoifCq31cSK7Hf/e5NaZrmbJAw7dKEpKyHeHDrjBGQynQXJu2zJx9B1VTNKIHE5EZpAdRol7bkFrZp8cQYq8W+v0p6QGctMGGEqvwLvSOBDpcw6gSJN1CjDvef6ftIopd0gcg2j7eEHUD8mBw0JSyCO7wlwZmWBqgT/eAmxlrHvOIY+0Lcv8LD6mDvpMljqI6D5PPNcvQVwV46GxM759HJGdrimrjG/Y6qqnnX+JGjrq366JH8yiGnvM4GZYFzzp9pmnYe402Od0223Iv9lw+vuCy9dQG8GUGIWC1Xh+/ywzQgwvYBV1rgQRUoZY5V8dA6eVwIUQzwr7jfecwr3tfzY3SWq9Qj9o6SWp7nQVNSipwE4HyooOt6iUyiQau8h/HRmFDoBq8kqNEfmeJQy6YXtMkkXvPyVkty7Hu+dXLit3BcsG/j44MiRKe4Rhjf41ElMCYTiIoKgtSjeP6ELRpM6RFo0ye5FrWMOX3wE6EN7g+YNhC4hL2epBN2kxEohEIPEBe8A08cz2wfnt92inIyIHHS7RPRZJWpaZrzAd4PBUK5fWoLdsU+DfuS4EZg7fh/iWGN7dS51gGzjcB0jTCmuC6q6mDG9VJ57geRRbkY7rZrVb3j4PJVXV29ESB9WOVz3HdbciN6ZRK5Ot79wfpisncNwygAaCNiyG5GVrT7XM0qW8e3OAkEL0+VA6nnArgG0Q1ShdNTQA2uxodIM8A/w3QNZIaWZJIH6YPTBv+vr1+2ZGgUV7vrNwAAAABJRU5ErkJggg==","smallIcon":"iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAABfklEQVR42mNggIL/DAyMDOSC/5WRif8rI378r4rYgM2g/w0OLP+rI4uAamJxGbAPKPkfjKsjZDHkq6NC4fI1MepYDIjyAtp+Dajg5v/K8Ev/a6JU0SzQAMp9AOK7/8uSeHF7JS2N9X9VVB9YcXWUP6ohEcv+V0VWEhcmVZFhQFs/Af3d/j80lPl/ZbQW0IATQFeu/F8WLUOcITXhmkBN1/9XRN4A0v/gYQAK7MrIeBJcAtQENgQYU1VR5UD+VyD+jTUwMQM3YgVyyFdXV1tcrkxZCRariqolbEB1+Faw4pJYbhC/oqJia29F8YN1lTn/v1THdBPjhUqQAZ8qY2dVVVWFna9JV31bGfPyaWX8/4bK8knEGBADMuBhZcL/45WpP4Hsv9DEtr2yomIJbo21kdrAxFUFVPwFZMib6tjOJRV5X/8Bo/J+VeLM+w0JHJWVlXn4Au8pNKDKYWLl5eX+QLwWiO8BNdsTcvpCoAGPQWkBTYoRaEBFYWEhJwO1AACRK/lksZaMlwAAAABJRU5ErkJggg=="},"description":"","helpHtml":"<div class=\"mtln\"><img src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAADMUlEQVR42t1XXUgUURjdfqnIyCio3sR8KIrCh+ilngrqwZdi2b2za4nVYvRgFo1z70w1RokFERL0UhCRYEUgZBYRFm710INP/aEUYVKCZBaiGe76dWYd1zuza+6uk0UDHzPsne+ec88933dnfb5/8SK/f87fAdaDARJKFwkWR0TJUIpymkdT8sksW5BdkggUAzSGICk6slEDoEtJVx7auUPEmZoFgeBpF7gdyqaM5+BKfUq+rmzOMJmpaQkYyvbMF8GephIIVmSWfNy/Egm9aUgMWN744wqMmSe8BmBXocYjJJ9BfEhOxNlFikTmTekBzp7YOfGsPJB+Qv8yeOC+tKIoifCq31cSK7Hf/e5NaZrmbJAw7dKEpKyHeHDrjBGQynQXJu2zJx9B1VTNKIHE5EZpAdRol7bkFrZp8cQYq8W+v0p6QGctMGGEqvwLvSOBDpcw6gSJN1CjDvef6ftIopd0gcg2j7eEHUD8mBw0JSyCO7wlwZmWBqgT/eAmxlrHvOIY+0Lcv8LD6mDvpMljqI6D5PPNcvQVwV46GxM759HJGdrimrjG/Y6qqnnX+JGjrq366JH8yiGnvM4GZYFzzp9pmnYe402Od0223Iv9lw+vuCy9dQG8GUGIWC1Xh+/ywzQgwvYBV1rgQRUoZY5V8dA6eVwIUQzwr7jfecwr3tfzY3SWq9Qj9o6SWp7nQVNSipwE4HyooOt6iUyiQau8h/HRmFDoBq8kqNEfmeJQy6YXtMkkXvPyVkty7Hu+dXLit3BcsG/j44MiRKe4Rhjf41ElMCYTiIoKgtSjeP6ELRpM6RFo0ye5FrWMOX3wE6EN7g+YNhC4hL2epBN2kxEohEIPEBe8A08cz2wfnt92inIyIHHS7RPRZJWpaZrzAd4PBUK5fWoLdsU+DfuS4EZg7fh/iWGN7dS51gGzjcB0jTCmuC6q6mDG9VJ57geRRbkY7rZrVb3j4PJVXV29ESB9WOVz3HdbciN6ZRK5Ot79wfpisncNwygAaCNiyG5GVrT7XM0qW8e3OAkEL0+VA6nnArgG0Q1ShdNTQA2uxodIM8A/w3QNZIaWZJIH6YPTBv+vr1+2ZGgUV7vrNwAAAABJRU5ErkJggg==\"><h3>HubSpot Incremental Load</h3><p style='white-space: pre-line; word-wrap: break-word;'></p><h3>Properties</h3><table><tbody><tr><th>Property</th><th>Type</th><th>Description</th></tr><tr><td>OAuth</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>The name of the OAuth entry to be used from the Matillion OAuth Manager to connect to the source. These must be set up in advance, using Project --> Mangage OAuth.</td></tr><tr><td>Tables and Columns</td><td>Grid</td><td style='white-space: pre-line; word-wrap: break-word;'>Contains the list of tables and columns (and in some cases an incremental_column to specify which column the load should be incremented on [accepts values of 0 or 1]) to be processed. </td></tr><tr><td>Connection Options</td><td>Grid</td><td style='white-space: pre-line; word-wrap: break-word;'>A list of values and parameters. Parameters and their allowed values are database/driver specific. Referring to the data model will provide insight of what you could provide here.\n<br>\nThey are usually not required as sensible defaults are assumed.</td></tr><tr><td>Cloud Storage Area</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>The URL and path of the target Google Storage bucket to be used for staging the queried data throughout the job.</td></tr><tr><td>Load Type</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>Sequential - Iterations are done in sequence, waiting for each to complete before starting the next. \nThis is the default.\n<br>\nConcurrent - Iterations are run concurrently. This requires all \"Variables to Iterate\" to be defined as\n<a href=\"https://redshiftsupport.matillion.com/customer/portal/articles/2037630?b_id=8915#header1\" target=\"_blank\"><i>Copied</i> variables</a>, so that each iteration gets its own copy of the variable isolated from the same\nvariable being used by other concurrent executions.\n<span class=\"env rs sf\"><br><b>Note:</b> The maximum concurrency is limited by the number of available threads (2x the number of virtual cpus on your cloud instance).</span></td></tr><tr><td>Stage Project</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>The project name where the staging data will be stored.</td></tr><tr><td>Stage Dataset</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>The dataset name where the staging data will be stored.</td></tr><tr><td>Stage Prefix</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>A prefix value that will be added to the start of the stage table names.\n<br>\ne.g. If a Stage Prefix of 'stage_' is specified and the table being processed is named 'test_data' then the target table will be named 'stage_test_data'.</td></tr><tr><td>Target Project</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>The project name where the target data will be stored.</td></tr><tr><td>Target Dataset</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>The dataset name where the target data will be stored.</td></tr><tr><td>Target Prefix</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>A prefix value that will be added to the start of the target table names.\n<br>\ne.g. If a Target Prefix of 'target_' is specified and the table being processed is named 'test_data' then the target table will be named 'target_test_data'.</td></tr></tbody></table></div>","created":1703067719617},"orchestrationJobs":{"HubSpot - 4 - Query Source":{"id":-2,"revision":1,"created":1702911350516,"timestamp":1702911350516,"components":{"27091":{"id":27091,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":-160,"y":0,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[27136],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Start","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27092":{"id":27092,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":1847161714,"x":160,"y":0,"width":32,"height":32,"inputConnectorIDs":[27098],"outputSuccessConnectorIDs":[27099],"outputFailureConnectorIDs":[27138],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"Message","mapTo":"error_message","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"Time Taken To Load","mapTo":"time_taken_to_load","validationStatus":null,"validationMessage":null},"3":{"slot":3,"fromId":null,"fromName":"Time Taken To Stage","mapTo":"time_taken_to_stage","validationStatus":null,"validationMessage":null},"4":{"slot":4,"fromId":null,"fromName":"Row Count","mapTo":"row_count","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Query Source"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Basic/Advanced Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Advanced"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Authentication","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${oauth}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Data Source","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Data Selection","elements":{},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Data Source Filter","elements":{},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Combine Filters","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":true,"status":{},"columnNames":[]},"10":{"slot":10,"name":"SQL Query","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${sql}"}}}},"visible":true,"status":{},"columnNames":[]},"11":{"slot":11,"name":"Limit","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"100"}}}},"visible":true,"status":{},"columnNames":[]},"12":{"slot":12,"name":"Connection Options","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"connectionOptions"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"parameter"},"2":{"slot":2,"type":"GRID","value":"value"}}}},"visible":true,"status":{},"columnNames":[]},"13":{"slot":13,"name":"Dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"14":{"slot":14,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${table}"}}}},"visible":true,"status":{},"columnNames":[]},"15":{"slot":15,"name":"Cloud Storage Staging Area","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${cloud_storage_area}"}}}},"visible":true,"status":{},"columnNames":[]},"16":{"slot":16,"name":"Load Options","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"On"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"On"}}}},"visible":true,"status":{},"columnNames":[]},"17":{"slot":17,"name":"Project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${project}"}}}},"visible":true,"status":{},"columnNames":[]},"20":{"slot":20,"name":"Data Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot"}}}},"visible":false,"status":{},"columnNames":[]},"1013":{"slot":1013,"name":"Auto Debug","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${autoDebug}"}}}},"visible":true,"status":{},"columnNames":[]},"1014":{"slot":1014,"name":"Debug Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${autoDebugVerbosity}"}}}},"visible":false,"status":{},"columnNames":[]},"1996":{"slot":1996,"name":"Location","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"gs://<bucket>/<path>"}}}},"visible":false,"status":{},"columnNames":[]},"1997":{"slot":1997,"name":"New Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"2000":{"slot":2000,"name":"Table Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Native"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27093":{"id":27093,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":166,"y":160,"width":32,"height":32,"inputConnectorIDs":[27138],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[27137],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Print Error"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"msg = \\\n\"\"\"There was an issue whilst retrieving data from the source. \nThe attempted query was: \\n{q}\nError: {e}\n\"\"\".format(q = sql, e = error_message) \nprint(msg)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27094":{"id":27094,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":160,"y":320,"width":32,"height":32,"inputConnectorIDs":[27137],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[27100],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Error"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"error"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Error Staging Data"},"2":{"slot":2,"type":"STRING","value":"${error_message}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27095":{"id":27095,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":320,"y":320,"width":32,"height":32,"inputConnectorIDs":[27100],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Failure"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27096":{"id":27096,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":320,"y":0,"width":32,"height":32,"inputConnectorIDs":[27099],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27097":{"id":27097,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":4,"y":3,"width":32,"height":32,"inputConnectorIDs":[27136],"outputSuccessConnectorIDs":[27098],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Limit HubSpot Data"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"# hard limit to HubSpot tables that can be set during testing (via `testlimit` connection option)\nif dataLimit:\n  sql += \" LIMIT {}\".format(dataLimit)  \n  context.updateVariable('sql', sql)\n  print(\"New SQL: {}\".format(sql))\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"27098":{"id":27098,"sourceID":27097,"targetID":27092},"27099":{"id":27099,"sourceID":27092,"targetID":27096}},"failureConnectors":{"27138":{"id":27138,"sourceID":27092,"targetID":27093}},"unconditionalConnectors":{"27100":{"id":27100,"sourceID":27094,"targetID":27095},"27136":{"id":27136,"sourceID":27091,"targetID":27097},"27137":{"id":27137,"sourceID":27093,"targetID":27094}},"trueConnectors":{},"falseConnectors":{},"iterationConnectors":{},"noteConnectors":{},"canUndo":false,"undoCommand":"","undoCreated":0,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{},"variables":{"sql":{"definition":{"name":"sql","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"oauth":{"definition":{"name":"oauth","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"table":{"definition":{"name":"table","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"schema":{"definition":{"name":"schema","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"dataset":{"definition":{"name":"dataset","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"project":{"definition":{"name":"project","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"database":{"definition":{"name":"database","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"autoDebug":{"definition":{"name":"autoDebug","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"Off"},"dataLimit":{"definition":{"name":"dataLimit","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"row_count":{"definition":{"name":"row_count","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"},"warehouse":{"definition":{"name":"warehouse","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"dist_style":{"definition":{"name":"dist_style","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"Even"},"kms_key_id":{"definition":{"name":"kms_key_id","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"staging_type":{"definition":{"name":"staging_type","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"Snowflake Managed"},"error_message":{"definition":{"name":"error_message","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"blob_container":{"definition":{"name":"blob_container","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"s3_bucket_name":{"definition":{"name":"s3_bucket_name","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"storage_account":{"definition":{"name":"storage_account","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"encryption_method":{"definition":{"name":"encryption_method","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"None"},"target_table_name":{"definition":{"name":"target_table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"autoDebugVerbosity":{"definition":{"name":"autoDebugVerbosity","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"3"},"cloud_storage_area":{"definition":{"name":"cloud_storage_area","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"time_taken_to_load":{"definition":{"name":"time_taken_to_load","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"},"time_taken_to_stage":{"definition":{"name":"time_taken_to_stage","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"}},"grids":{"connectionOptions":{"definition":{"name":"connectionOptions","scope":"BRANCH","definitions":[{"name":"parameter","type":"TEXT"},{"name":"value","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]}}},"HubSpot - 2 - Wrapper":{"id":-2,"revision":1,"created":1702911350516,"timestamp":1702911350516,"components":{"27178":{"id":27178,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":-240,"y":-80,"width":32,"height":32,"inputConnectorIDs":[27187],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[27189],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log End"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"info"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Iteration Duration"},"2":{"slot":2,"type":"STRING","value":"${iter_duration}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"Iteration End"},"2":{"slot":2,"type":"STRING","value":"${iter_state}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"Iteration Name"},"2":{"slot":2,"type":"STRING","value":"${table_name}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27179":{"id":27179,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":-400,"y":-160,"width":32,"height":32,"inputConnectorIDs":[27186],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Iteration Success"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27180":{"id":27180,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":80,"y":-80,"width":32,"height":32,"inputConnectorIDs":[27191],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Iteration Failure: Error Cleaning Stage Tables"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27181":{"id":27181,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":-624,"y":0,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[27188],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Start","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27182":{"id":27182,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":-400,"y":0,"width":32,"height":32,"inputConnectorIDs":[27188],"outputSuccessConnectorIDs":[27186],"outputFailureConnectorIDs":[27190,27192],"outputUnconditionalConnectorIDs":[27187],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"metadata_stage_table_name","mapTo":"metadata_stage_table_name","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"comp_message","mapTo":"iter_component_message","validationStatus":null,"validationMessage":null},"3":{"slot":3,"fromId":null,"fromName":"comp_name","mapTo":"iter_component_name","validationStatus":null,"validationMessage":null},"4":{"slot":4,"fromId":null,"fromName":"Status","mapTo":"iter_state","validationStatus":null,"validationMessage":null},"5":{"slot":5,"fromId":null,"fromName":"target_table_name","mapTo":"target_table_name","validationStatus":null,"validationMessage":null},"6":{"slot":6,"fromId":null,"fromName":"Duration","mapTo":"iter_duration","validationStatus":null,"validationMessage":null},"7":{"slot":7,"fromId":null,"fromName":"stage_table_name","mapTo":"stage_table_name","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Stage And Load"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot - 3 - Stage and Load"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"staging_type"},"2":{"slot":2,"type":"STRING","value":"${staging_type}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"blob_container"},"2":{"slot":2,"type":"STRING","value":"${blob_container}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"cloud_storage_area"},"2":{"slot":2,"type":"STRING","value":"${cloud_storage_area}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"dist_style"},"2":{"slot":2,"type":"STRING","value":"${dist_style}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"encryption_method"},"2":{"slot":2,"type":"STRING","value":"${encryption_method}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"kms_key_id"},"2":{"slot":2,"type":"STRING","value":"${kms_key_id}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"oauth"},"2":{"slot":2,"type":"STRING","value":"${oauth}"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"s3_bucket_name"},"2":{"slot":2,"type":"STRING","value":"${s3_bucket_name}"}}},"9":{"slot":9,"values":{"1":{"slot":1,"type":"STRING","value":"stage_database"},"2":{"slot":2,"type":"STRING","value":"${stage_database}"}}},"10":{"slot":10,"values":{"1":{"slot":1,"type":"STRING","value":"stage_dataset"},"2":{"slot":2,"type":"STRING","value":"${stage_dataset}"}}},"11":{"slot":11,"values":{"1":{"slot":1,"type":"STRING","value":"stage_prefix"},"2":{"slot":2,"type":"STRING","value":"${stage_prefix}"}}},"12":{"slot":12,"values":{"1":{"slot":1,"type":"STRING","value":"stage_project"},"2":{"slot":2,"type":"STRING","value":"${stage_project}"}}},"13":{"slot":13,"values":{"1":{"slot":1,"type":"STRING","value":"stage_warehouse"},"2":{"slot":2,"type":"STRING","value":"${stage_warehouse}"}}},"14":{"slot":14,"values":{"1":{"slot":1,"type":"STRING","value":"stage_schema"},"2":{"slot":2,"type":"STRING","value":"${stage_schema}"}}},"15":{"slot":15,"values":{"1":{"slot":1,"type":"STRING","value":"target_database"},"2":{"slot":2,"type":"STRING","value":"${target_database}"}}},"16":{"slot":16,"values":{"1":{"slot":1,"type":"STRING","value":"target_dataset"},"2":{"slot":2,"type":"STRING","value":"${target_dataset}"}}},"17":{"slot":17,"values":{"1":{"slot":1,"type":"STRING","value":"target_prefix"},"2":{"slot":2,"type":"STRING","value":"${target_prefix}"}}},"18":{"slot":18,"values":{"1":{"slot":1,"type":"STRING","value":"target_project"},"2":{"slot":2,"type":"STRING","value":"${target_project}"}}},"19":{"slot":19,"values":{"1":{"slot":1,"type":"STRING","value":"target_warehouse"},"2":{"slot":2,"type":"STRING","value":"${target_warehouse}"}}},"20":{"slot":20,"values":{"1":{"slot":1,"type":"STRING","value":"target_schema"},"2":{"slot":2,"type":"STRING","value":"${target_schema}"}}},"21":{"slot":21,"values":{"1":{"slot":1,"type":"STRING","value":"storage_account"},"2":{"slot":2,"type":"STRING","value":"${storage_account}"}}},"22":{"slot":22,"values":{"1":{"slot":1,"type":"STRING","value":"table_name"},"2":{"slot":2,"type":"STRING","value":"${table_name}"}}},"23":{"slot":23,"values":{"1":{"slot":1,"type":"STRING","value":"autoDebug"},"2":{"slot":2,"type":"STRING","value":"${autoDebug}"}}},"24":{"slot":24,"values":{"1":{"slot":1,"type":"STRING","value":"autoDebugVerbosity"},"2":{"slot":2,"type":"STRING","value":"${autoDebugVerbosity}"}}},"25":{"slot":25,"values":{"1":{"slot":1,"type":"STRING","value":"testLimit"},"2":{"slot":2,"type":"STRING","value":"${testLimit}"}}},"26":{"slot":26,"values":{"1":{"slot":1,"type":"STRING","value":"target_platform"},"2":{"slot":2,"type":"STRING","value":"${target_platform}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"connection_options"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"param"},"2":{"slot":2,"type":"GRID","value":"value"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"connection_options"},"2":{"slot":2,"type":"STRING","value":"grid"}}},"2":{"slot":2,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"source_list"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"tableName"},"2":{"slot":2,"type":"GRID","value":"columnName"},"3":{"slot":3,"type":"GRID","value":"isIncremental"},"4":{"slot":4,"type":"GRID","value":"isKey"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"source_list"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot Incremental Load"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Incremental"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27183":{"id":27183,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1291941950,"x":-80,"y":-80,"width":32,"height":32,"inputConnectorIDs":[27189],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[27191],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Drop Stage Tables"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_project}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${metadata_stage_table_name}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Ignore Missing","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Yes"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27184":{"id":27184,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":-399,"y":161,"width":32,"height":32,"inputConnectorIDs":[27190],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Iteration Failure"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27185":{"id":27185,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":-240,"y":80,"width":32,"height":32,"inputConnectorIDs":[27192],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Failure"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"error"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Iteration Error"},"2":{"slot":2,"type":"STRING","value":"${iter_component_message ? iter_component_message : \" \"}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"Iteration Error At"},"2":{"slot":2,"type":"STRING","value":"${iter_component_name ? iter_component_name: \" \"}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"27186":{"id":27186,"sourceID":27182,"targetID":27179}},"failureConnectors":{"27190":{"id":27190,"sourceID":27182,"targetID":27184},"27191":{"id":27191,"sourceID":27183,"targetID":27180},"27192":{"id":27192,"sourceID":27182,"targetID":27185}},"unconditionalConnectors":{"27187":{"id":27187,"sourceID":27182,"targetID":27178},"27188":{"id":27188,"sourceID":27181,"targetID":27182},"27189":{"id":27189,"sourceID":27178,"targetID":27183}},"trueConnectors":{},"falseConnectors":{},"iterationConnectors":{},"noteConnectors":{},"canUndo":false,"undoCommand":"","undoCreated":0,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"27177":{"id":27177,"x":-795,"y":-398,"width":952,"height":635,"text":"**Execute ETL Flow for specific source table**\n\nThis job wraps our Stage 3 job (Stage Object). This is done so that we can manage some exported variables from the Stage 3. If we were directly iterating over (Stage Object) we would not be able to handle these exported variables.\n\n- (__Stage And Load__) Passes in the parameters and their values into the Stage - 3 job. Regardless of the result, the __metadata_stage_table_name__ and __stage_table_name__ are exported - this is to ensure these tables are dropped if the process fails before it has time to clean up these staging tables (when a run succeeds, these staging tables will no longer exist).\n\n- (__Drop Stage Tables__) Passes in the exported table names to be deleted. This flow is executed regardless of the result of Stage Object.","colour":"e6e63c"}},"variables":{"oauth":{"definition":{"name":"oauth","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"autoDebug":{"definition":{"name":"autoDebug","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"testLimit":{"definition":{"name":"testLimit","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"dist_style":{"definition":{"name":"dist_style","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"iter_state":{"definition":{"name":"iter_state","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"kms_key_id":{"definition":{"name":"kms_key_id","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"table_name":{"definition":{"name":"table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"log_metrics":{"definition":{"name":"log_metrics","type":"TEXT","scope":"TASKBATCH","description":"TRUE or FALSE (default=TRUE).\nWhether or not to add useful information to the metrics store.\n","visibility":"PUBLIC"},"value":""},"stage_prefix":{"definition":{"name":"stage_prefix","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"stage_schema":{"definition":{"name":"stage_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"staging_type":{"definition":{"name":"staging_type","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"error_message":{"definition":{"name":"error_message","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"iter_duration":{"definition":{"name":"iter_duration","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"0"},"source_schema":{"definition":{"name":"source_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"stage_dataset":{"definition":{"name":"stage_dataset","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"stage_project":{"definition":{"name":"stage_project","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_prefix":{"definition":{"name":"target_prefix","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_schema":{"definition":{"name":"target_schema","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"blob_container":{"definition":{"name":"blob_container","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"s3_bucket_name":{"definition":{"name":"s3_bucket_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"stage_database":{"definition":{"name":"stage_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_dataset":{"definition":{"name":"target_dataset","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_project":{"definition":{"name":"target_project","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"stage_warehouse":{"definition":{"name":"stage_warehouse","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"storage_account":{"definition":{"name":"storage_account","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_database":{"definition":{"name":"target_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_platform":{"definition":{"name":"target_platform","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_table_name":{"definition":{"name":"stage_table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"target_warehouse":{"definition":{"name":"target_warehouse","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"encryption_method":{"definition":{"name":"encryption_method","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_table_name":{"definition":{"name":"target_table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"autoDebugVerbosity":{"definition":{"name":"autoDebugVerbosity","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"cloud_storage_area":{"definition":{"name":"cloud_storage_area","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"iter_component_name":{"definition":{"name":"iter_component_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"limitedSQLStatement":{"definition":{"name":"limitedSQLStatement","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"iter_component_message":{"definition":{"name":"iter_component_message","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"password_manager_entry":{"definition":{"name":"password_manager_entry","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"metadata_stage_table_name":{"definition":{"name":"metadata_stage_table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""}},"grids":{"message":{"definition":{"name":"message","scope":"BRANCH","definitions":[{"name":"key","type":"TEXT"},{"name":"value","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[{"values":["Iteration Duration","${iter_duration}"]},{"values":["Iteration End","${iter_state}"]}]},"source_list":{"definition":{"name":"source_list","scope":"BRANCH","definitions":[{"name":"tableName","type":"TEXT"},{"name":"columnName","type":"TEXT"},{"name":"isIncremental","type":"DECIMAL"},{"name":"isKey","type":"DECIMAL"}],"description":"","visibility":"PUBLIC"},"values":[]},"connection_options":{"definition":{"name":"connection_options","scope":"BRANCH","definitions":[{"name":"param","type":"TEXT"},{"name":"value","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]}}},"HubSpot - 1 - Orchestrator":{"id":-2,"revision":1,"created":1702911350516,"timestamp":1702911350516,"components":{"27101":{"id":27101,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":640,"y":4,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"2 - Wrapper"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot - 2 - Wrapper"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"table_name"},"2":{"slot":2,"type":"STRING","value":"${table_name}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"oauth"},"2":{"slot":2,"type":"STRING","value":"${oauth}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"s3_bucket_name"},"2":{"slot":2,"type":"STRING","value":"${s3_bucket_name}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"stage_prefix"},"2":{"slot":2,"type":"STRING","value":"${stage_prefix}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"stage_schema"},"2":{"slot":2,"type":"STRING","value":"${stage_schema}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"target_prefix"},"2":{"slot":2,"type":"STRING","value":"${target_prefix}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"target_schema"},"2":{"slot":2,"type":"STRING","value":"${target_schema}"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"dist_style"},"2":{"slot":2,"type":"STRING","value":"${dist_style}"}}},"9":{"slot":9,"values":{"1":{"slot":1,"type":"STRING","value":"target_platform"},"2":{"slot":2,"type":"STRING","value":"${target_platform}"}}},"10":{"slot":10,"values":{"1":{"slot":1,"type":"STRING","value":"stage_database"},"2":{"slot":2,"type":"STRING","value":"${stage_database}"}}},"11":{"slot":11,"values":{"1":{"slot":1,"type":"STRING","value":"stage_project"},"2":{"slot":2,"type":"STRING","value":"${stage_project}"}}},"12":{"slot":12,"values":{"1":{"slot":1,"type":"STRING","value":"stage_warehouse"},"2":{"slot":2,"type":"STRING","value":"${stage_warehouse}"}}},"13":{"slot":13,"values":{"1":{"slot":1,"type":"STRING","value":"stage_dataset"},"2":{"slot":2,"type":"STRING","value":"${stage_dataset}"}}},"14":{"slot":14,"values":{"1":{"slot":1,"type":"STRING","value":"cloud_storage_area"},"2":{"slot":2,"type":"STRING","value":"${cloud_storage_area}"}}},"15":{"slot":15,"values":{"1":{"slot":1,"type":"STRING","value":"target_database"},"2":{"slot":2,"type":"STRING","value":"${target_database}"}}},"16":{"slot":16,"values":{"1":{"slot":1,"type":"STRING","value":"target_dataset"},"2":{"slot":2,"type":"STRING","value":"${target_dataset}"}}},"17":{"slot":17,"values":{"1":{"slot":1,"type":"STRING","value":"target_project"},"2":{"slot":2,"type":"STRING","value":"${target_project}"}}},"18":{"slot":18,"values":{"1":{"slot":1,"type":"STRING","value":"target_warehouse"},"2":{"slot":2,"type":"STRING","value":"${target_warehouse}"}}},"19":{"slot":19,"values":{"1":{"slot":1,"type":"STRING","value":"blob_container"},"2":{"slot":2,"type":"STRING","value":"${blob_container}"}}},"20":{"slot":20,"values":{"1":{"slot":1,"type":"STRING","value":"encryption_method"},"2":{"slot":2,"type":"STRING","value":"${encryption_method}"}}},"21":{"slot":21,"values":{"1":{"slot":1,"type":"STRING","value":"kms_key_id"},"2":{"slot":2,"type":"STRING","value":"${kms_key_id}"}}},"22":{"slot":22,"values":{"1":{"slot":1,"type":"STRING","value":"staging_type"},"2":{"slot":2,"type":"STRING","value":"${staging_type}"}}},"23":{"slot":23,"values":{"1":{"slot":1,"type":"STRING","value":"autoDebug"},"2":{"slot":2,"type":"STRING","value":"${autoDebug}"}}},"24":{"slot":24,"values":{"1":{"slot":1,"type":"STRING","value":"autoDebugVerbosity"},"2":{"slot":2,"type":"STRING","value":"${autoDebugVerbosity}"}}},"25":{"slot":25,"values":{"1":{"slot":1,"type":"STRING","value":"testLimit"},"2":{"slot":2,"type":"STRING","value":"${testLimit}"}}},"26":{"slot":26,"values":{"1":{"slot":1,"type":"STRING","value":"storage_account"},"2":{"slot":2,"type":"STRING","value":"${storage_account}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"advanced_connection_options"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"parameter"},"2":{"slot":2,"type":"GRID","value":"value"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"connection_options"},"2":{"slot":2,"type":"STRING","value":"grid"}}},"2":{"slot":2,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"internal_source_list"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"tableName"},"2":{"slot":2,"type":"GRID","value":"columnName"},"3":{"slot":3,"type":"GRID","value":"incremental"},"4":{"slot":4,"type":"GRID","value":"key"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"source_list"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot Incremental Load"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Incremental"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[27112]},"27102":{"id":27102,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":156,"y":-7,"width":32,"height":32,"inputConnectorIDs":[27104],"outputSuccessConnectorIDs":[27105],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Magic Variables"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"# this may or may not contain variables which are not valid connection options\n# and are designed to be used elsewhere within the shared job, they need to be removed from the grid \n# to prevent downstream errors \nopts = context.getGridVariable('advanced_connection_options')\n\n# create a new list to store the real connection options \nfiltered = [] \n\n# magic vars is a map: lower case param name => job variable name \nmagic_vars = {\n\t'autodebug': 'autoDebug', # passed into XX Query Components for CData Debugging \n    'verbosity': 'autoDebugVerbosity', # passed into XX Query Components (default = 3)\n    'testlimit': 'testLimit', # HubSpot specific var for limiting the amount of rows returned \n}\n\nif opts: \n  for r in opts:\n    param = r[0]\n    val = r[1]\n    \n    p = param.lower()\n    # check if this is a magic variable \n    # update the underlying job variable if it is \n    if p in magic_vars:\n      context.updateVariable(magic_vars[p], val)\n    # otherwise, its a normal connection option \n    else:\n      filtered.append( [param, val] )\n    \n  context.updateGridVariable('advanced_connection_options', filtered)\n  \n  \n  \n  "}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27103":{"id":27103,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":800,"y":-80,"width":32,"height":32,"inputConnectorIDs":[27109],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Pipeline End"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"info"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Pipeline Duration"},"2":{"slot":2,"type":"STRING","value":"${logging_duration}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"Pipeline End"},"2":{"slot":2,"type":"STRING","value":"${status}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27144":{"id":27144,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":3,"y":6,"width":32,"height":32,"inputConnectorIDs":[27110],"outputSuccessConnectorIDs":[27104],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Pipeline Start"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"info"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Pipeline Started"},"2":{"slot":2,"type":"STRING","value":" "}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27145":{"id":27145,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":480,"y":0,"width":32,"height":32,"inputConnectorIDs":[27106],"outputSuccessConnectorIDs":[27107],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Update [Environment Default]"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"# Script to replace any [Environment Default] entries with the schema name\nif target_platform == \"Redshift\":\n\n    if stage_schema == \"[Environment Default]\":\n        context.updateVariable(\"stage_schema\", environment_default_schema)\n        print(\n            f\"Updating stage_schema from [Environment Default] to: {environment_default_schema}\"\n        )\n\n    if target_schema == \"[Environment Default]\":\n        context.updateVariable(\"target_schema\", environment_default_schema)\n        print(\n            f\"Updating target_schema from [Environment Default] to: {environment_default_schema}\"\n        )\n\nelif target_platform == \"Snowflake\":\n\n    if target_warehouse == \"[Environment Default]\":\n        context.updateVariable(\"target_warehouse\", environment_port)\n        print(\n            f\"Updating target_warehouse from [Environment Default] to: {environment_port}\"\n        )\n\n    if target_database == \"[Environment Default]\":\n        context.updateVariable(\"target_database\", environment_database)\n        print(\n            f\"Updating target_database from [Environment Default] to: {environment_database}\"\n        )\n\n    if target_schema == \"[Environment Default]\":\n        context.updateVariable(\"target_schema\", environment_default_schema)\n        print(\n            f\"Updating target_schema from [Environment Default] to: {environment_default_schema}\"\n        )\n\n    if stage_warehouse == \"[Environment Default]\":\n        context.updateVariable(\"stage_warehouse\", environment_port)\n        print(\n            f\"Updating stage_warehouse from [Environment Default] to: {environment_port}\"\n        )\n\n    if stage_database == \"[Environment Default]\":\n        context.updateVariable(\"stage_database\", environment_database)\n        print(\n            f\"Updating stage_database from [Environment Default] to: {environment_database}\"\n        )\n\n    if stage_schema == \"[Environment Default]\":\n        context.updateVariable(\"stage_schema\", environment_default_schema)\n        print(\n            f\"Updating stage_schema from [Environment Default] to: {environment_default_schema}\"\n        )\n\nelif target_platform == \"BigQuery\":\n\n    if stage_dataset == \"[Environment Default]\":\n        context.updateVariable(\"stage_dataset\", environment_default_schema)\n        print(\n            f\"Updating stage_dataset from [Environment Default] to: {environment_default_schema}\"\n        )\n\n    if target_dataset == \"[Environment Default]\":\n        context.updateVariable(\"target_dataset\", environment_default_schema)\n        print(\n            f\"Updating target_dataset from [Environment Default] to: {environment_default_schema}\"\n        )\n\n    if stage_project == \"[Environment Default]\":\n        context.updateVariable(\"stage_project\", environment_database)\n        print(\n            f\"Updating stage_project from [Environment Default] to: {environment_default_schema}\"\n        )\n\n    if target_project == \"[Environment Default]\":\n        context.updateVariable(\"target_project\", environment_database)\n        print(\n            f\"Updating target_project from [Environment Default] to: {environment_default_schema}\"\n        )\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27146":{"id":27146,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":800,"y":0,"width":32,"height":32,"inputConnectorIDs":[27108],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Fin"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27147":{"id":27147,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":800,"y":80,"width":32,"height":32,"inputConnectorIDs":[27111],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Failure"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27148":{"id":27148,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":-160,"y":0,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[27110],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27149":{"id":27149,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1255297039,"x":320,"y":0,"width":32,"height":32,"inputConnectorIDs":[27105],"outputSuccessConnectorIDs":[27106],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Get Unique Table Names"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"tbls = list(set([str(x[0]) for x in context.getGridVariable('source_list')]))\nunique_tbls = [[x] for x in tbls]\n\n# set the Grid Variable \nprint(\"Updated GridVariable 'DistinctTables' with: {}\".format(unique_tbls))\ncontext.updateGridVariable(\"DistinctTables\",unique_tbls)\n\n\ninternal_source_list = [[x[0], x[1], '0', '0'] for x in context.getGridVariable('source_list')]\ncontext.updateGridVariable('internal_source_list', internal_source_list)\n\n\n\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27150":{"id":27150,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"ITERATE","implementationID":-424773870,"x":640,"y":4,"width":32,"height":16,"inputConnectorIDs":[27107],"outputSuccessConnectorIDs":[27108],"outputFailureConnectorIDs":[27111],"outputUnconditionalConnectorIDs":[27109],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"Duration","mapTo":"logging_duration","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"Status","mapTo":"status","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Iterate Through Tables"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Grid Variable","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DistinctTables"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Grid Variable Mapping","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"table_name"},"2":{"slot":2,"type":"STRING","value":"table_name"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Break on Failure","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"No"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Concurrency","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${load_concurrent}"}}}},"visible":true,"status":{},"columnNames":[]},"999":{"slot":999,"name":"Record Values In Task History","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Yes"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[27112],"inputIterationConnectorIDs":[]}},"successConnectors":{"27104":{"id":27104,"sourceID":27144,"targetID":27102},"27105":{"id":27105,"sourceID":27102,"targetID":27149},"27106":{"id":27106,"sourceID":27149,"targetID":27145},"27107":{"id":27107,"sourceID":27145,"targetID":27150},"27108":{"id":27108,"sourceID":27150,"targetID":27146}},"failureConnectors":{"27111":{"id":27111,"sourceID":27150,"targetID":27147}},"unconditionalConnectors":{"27109":{"id":27109,"sourceID":27150,"targetID":27103},"27110":{"id":27110,"sourceID":27148,"targetID":27144}},"trueConnectors":{},"falseConnectors":{},"iterationConnectors":{"27112":{"id":27112,"sourceID":27150,"targetID":27101}},"noteConnectors":{},"canUndo":false,"undoCommand":"","undoCreated":0,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"27140":{"id":27140,"x":1665,"y":-573,"width":250,"height":58,"text":"** IGNORE **","colour":"d60000"},"27141":{"id":27141,"x":-1389,"y":376,"width":250,"height":58,"text":"** IGNORE **","colour":"d60000"},"27142":{"id":27142,"x":-791,"y":-471,"width":624,"height":388,"text":"**HubSpot Incremental Load** \n\nRoot (Stage 1) Orchestration Job of the Matillion Incremental Load pattern. \nThis job is the entry point of the shared job and is responsible for validating and reformatting some of the supplied data.\nThe pattern aims to synchronize INSERTS and UPDATES in a particular source, to a table within the relevant Cloud Data Warehouse.\n\n**Pattern Overview**\n\nThere are 4 distinct stages, each stage is represented by an orchestration job using the naming convention:\n__**<Source Name> - <Stage Number> - <Stage Name>**__\n\n+ **Stage 1** -  performs input validation and prepares the variables for **Stage 3**\n\n+ **Stage 2** - wrapper for **Stage 3** to ensure that in the case of any errors, relevant staging tables are nicely cleaned up.\n\n+ **Stage 3** - Orchestration job containing the ETL logic for a single source table.\n\n+ **Stage 4** - wrapper orchestration job to Query the underlying source. This is abstracted away from the rest of the logic to make the pattern more reusable.\n\nMore detailed information and specifics relating to each connector can be found within each job.\n\n\n======================================================================================================================================================================================================","colour":"00ce4f"},"27143":{"id":27143,"x":-75,"y":-233,"width":789,"height":310,"text":"** Prepare User Input **\n\n- (__Log Pipeline Start__) Writes to the metrics table.\n\n- (__Magic Variables__) Script that extracts some special variables from connection options. These are used internally to allow us to use 'hidden' variables that will not affect the slots of the shared job. Currently, this is used to switch on CData logging.\n\n- (__Get Unique Table Names__) Script to extract a list of table names from the source. This list is then iterated over, and the same ETL logic is applied to each source. This information is stored in the __distinct_table_names__ grid variable.\n\n- (__Update [Environment Default]__) Script to replace any [Environment Default] values with their actual value.\n\n- (__2 - Wrapper__) Iterate over each of the __distinct_table_names__ and call the lower level ETL process for each table in the source.","colour":"e6e63c"}},"variables":{"oauth":{"definition":{"name":"oauth","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":null},"status":{"definition":{"name":"status","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"autoDebug":{"definition":{"name":"autoDebug","type":"TEXT","scope":"BRANCH","description":"","visibility":"PRIVATE"},"value":"Off"},"testLimit":{"definition":{"name":"testLimit","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"dist_style":{"definition":{"name":"dist_style","type":"TEXT","scope":"BRANCH","description":"The distribution style to be used for both the staged and target data.\nThe options are:\nEVEN: Distribute data across all nodes evenly.\nALL: Distribute a full copy of the data onto each node.\nNote: KEY distribution is not implemented.","visibility":"PRIVATE"},"value":null},"kms_key_id":{"definition":{"name":"kms_key_id","type":"TEXT","scope":"BRANCH","description":"","visibility":"PRIVATE"},"value":null},"table_name":{"definition":{"name":"table_name","type":"TEXT","scope":"BRANCH","description":"","visibility":"PRIVATE"},"value":""},"stage_prefix":{"definition":{"name":"stage_prefix","type":"TEXT","scope":"BRANCH","description":"The prefix to assign to each of the source tables when they are staged. This is user assigned to help avoid any potential clashes with existing tables.\nE.g. If stage_prefix is set to 'stage_' and the table being processed is named 'test_data' then the staged table will be stored as 'stage_test_data'.","visibility":"PUBLIC"},"value":null},"stage_schema":{"definition":{"name":"stage_schema","type":"TEXT","scope":"BRANCH","description":"The schema where the staged data will be stored.","visibility":"PRIVATE"},"value":null},"staging_type":{"definition":{"name":"staging_type","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"Snowflake Managed"},"stage_dataset":{"definition":{"name":"stage_dataset","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":null},"stage_project":{"definition":{"name":"stage_project","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_prefix":{"definition":{"name":"target_prefix","type":"TEXT","scope":"BRANCH","description":"The prefix to assign to each of the target tables when they are created/updated. This is user assigned to help avoid any potential clashes with existing tables.\nE.g. If target_prefix is set to 'target_' and the table being processed is named 'test_data' then the staged table will be stored as 'target_test_data'.","visibility":"PUBLIC"},"value":null},"target_schema":{"definition":{"name":"target_schema","type":"TEXT","scope":"BRANCH","description":"The schema where the target data will be stored.","visibility":"PRIVATE"},"value":null},"blob_container":{"definition":{"name":"blob_container","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"s3_bucket_name":{"definition":{"name":"s3_bucket_name","type":"TEXT","scope":"BRANCH","description":"The name of the AWS S3 bucket that will be used for staging the source data.","visibility":"PRIVATE"},"value":null},"stage_database":{"definition":{"name":"stage_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"target_dataset":{"definition":{"name":"target_dataset","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"target_project":{"definition":{"name":"target_project","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"load_concurrent":{"definition":{"name":"load_concurrent","type":"TEXT","scope":"BRANCH","description":"Determines whether the tables being processed should be processed one after the other (Sequential) or at the same time (Concurrent).","visibility":"PUBLIC"},"value":"Sequential"},"stage_warehouse":{"definition":{"name":"stage_warehouse","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"storage_account":{"definition":{"name":"storage_account","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"target_database":{"definition":{"name":"target_database","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":null},"target_platform":{"definition":{"name":"target_platform","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"BigQuery"},"logging_duration":{"definition":{"name":"logging_duration","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"target_warehouse":{"definition":{"name":"target_warehouse","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"encryption_method":{"definition":{"name":"encryption_method","type":"TEXT","scope":"BRANCH","description":"","visibility":"PRIVATE"},"value":"None"},"autoDebugVerbosity":{"definition":{"name":"autoDebugVerbosity","type":"TEXT","scope":"BRANCH","description":"","visibility":"PRIVATE"},"value":"3"},"cloud_storage_area":{"definition":{"name":"cloud_storage_area","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":null},"password_manager_entry":{"definition":{"name":"password_manager_entry","type":"TEXT","scope":"BRANCH","description":"The entry name in the password manager to use for the source database connection. This will need to be set up separately from the shared job.\nNote: This applies to the Generic JDBC routine. Different authentication options may be required for the other routines.","visibility":"PRIVATE"},"value":""}},"grids":{"source_list":{"definition":{"name":"source_list","scope":"BRANCH","definitions":[{"name":"table","type":"TEXT"},{"name":"column","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]},"DistinctTables":{"definition":{"name":"DistinctTables","scope":"BRANCH","definitions":[{"name":"table_name","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[{"values":["HubSpotDummyTableIgnoreThis"]}]},"internal_source_list":{"definition":{"name":"internal_source_list","scope":"BRANCH","definitions":[{"name":"tableName","type":"TEXT"},{"name":"columnName","type":"TEXT"},{"name":"incremental","type":"DECIMAL"},{"name":"key","type":"DECIMAL"}],"description":"","visibility":"PRIVATE"},"values":[]},"advanced_connection_options":{"definition":{"name":"advanced_connection_options","scope":"BRANCH","definitions":[{"name":"parameter","type":"TEXT"},{"name":"value","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]}}},"HubSpot - 3 - Stage and Load":{"id":-2,"revision":4,"created":1702996864832,"timestamp":1702996864832,"components":{"27120":{"id":27120,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":2000,"y":0,"width":32,"height":32,"inputConnectorIDs":[27166],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[27171],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Extract Start"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"info"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Extract Start"},"2":{"slot":2,"type":"STRING","value":"${load_strategy}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27121":{"id":27121,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":2160,"y":-160,"width":32,"height":32,"inputConnectorIDs":[27173],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Extract End"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"info"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Extract Stage Duration"},"2":{"slot":2,"type":"STRING","value":"${time_taken_to_stage ? time_taken_to_stage : 0}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"Extract Load Duration"},"2":{"slot":2,"type":"STRING","value":"${time_taken_to_load ? time_taken_to_load : 0}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"Extract Row Count"},"2":{"slot":2,"type":"STRING","value":"${static_row_count}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"Extract End"},"2":{"slot":2,"type":"STRING","value":"${extract_state}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"Extract Duration"},"2":{"slot":2,"type":"STRING","value":"${stage_duration}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27122":{"id":27122,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":2160,"y":160,"width":32,"height":32,"inputConnectorIDs":[27174],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End - Extract Failure"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27123":{"id":27123,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1255297039,"x":880,"y":0,"width":32,"height":32,"inputConnectorIDs":[27164],"outputSuccessConnectorIDs":[27157],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"Component","mapTo":"comp_name","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"Message","mapTo":"comp_message","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Adjust Source List"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"def truncate(word, length = 12):\n  \"\"\"\n  `truncate()` will convert items to a string and ensure they are the length specified by user \n  Used within stringer function to ensure all columns are the same width \n  \"\"\"\n  if len(word.strip()) <= length:\n    return word\n  return \"{}...\".format( word[:(length-3)] )\n  \ndef stringer(lst, colwidth = 30):\n  \"\"\"\n  `stringer()` is used for printing Grid Variables in a more readable manner.\n  \n  It will handle formatting the grid variable to look more like how it is presented in the product \n  (as a table) \n  \"\"\"\n  msgs = [] \n  for row in lst:\n    msg = \" | \".join(\n    \t[\n        \"{:<{width}}\".format( truncate(str(x), length = colwidth), width = colwidth)\n        for x in row \n        ]\n    )\n    msgs.append(msg)\n    \n  return \"\\n\" + \"\\n\".join(msgs)\n\nsource_list = context.getGridVariable('source_list')\nprint(\"Grid variable 'source_list': {}\".format(stringer(source_list)))\n        \nsource_table_metadata = context.getGridVariable('source_table_metadata')\nprint(\"\\nGrid variable 'source_table_metadata': {}\".format(stringer(source_table_metadata)))\n\ninc_col_options = ['Last Modified Date', 'Last modified date', 'CreatedAt', 'TimeAddedToList', 'SavedAt']\nprint(\"\\nVariable 'inc_col_options': {}\".format(inc_col_options))\n\n# is_table() checks that a row of a grid variable belongs to the table we \n# are currently iterating over\nis_table = lambda x: x[0] == table_name\n\n# filter source_list to current table and extract list of columns\nsrc = [ x for x in source_list if is_table(x) ] \nrequested_columns = [ x[1] for x in src ]\n\n# extract key and inc columns\n# Note that the true/false values for IsKey are sometimes truncated to three letters, i.e. tru/fal\nkey_columns = [ x[0] for x in source_table_metadata if x[6].lower().startswith('t')]\n\n# loop through ordered list of possible incremental columns, using the first one found\nmetadata_columns = [ x[0] for x in source_table_metadata ]\ninc_columns = []\nfor i in inc_col_options:\n  if i in metadata_columns:\n    inc_columns = [i]\n    break\n\n# add missing columns, preserving order set by user\nall_columns = requested_columns\n\nfor x in key_columns:\n    if x not in all_columns:\n        all_columns.append(x)\n\nfor x in inc_columns:\n    if x not in all_columns:\n        all_columns.append(x)   \n        \n# build new source column list with inc and key col flags\nnew_src = [] \n\nfor c in all_columns:\n    new_src.append(\n        [\n            table_name, \n            c,\n            '1' if c in inc_columns else '0',\n            '1' if c in key_columns else '0'\n  \t\t]\n  \t)\n    \ncontext.updateGridVariable('source_list', new_src)\nprint(\"\\nUpdated grid variable 'source_list': {}\".format(stringer(new_src)))\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27124":{"id":27124,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":0,"y":0,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[27172],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Start","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27125":{"id":27125,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":480,"y":0,"width":32,"height":32,"inputConnectorIDs":[27168],"outputSuccessConnectorIDs":[27165],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"Component","mapTo":"comp_name","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"error_message","mapTo":"comp_message","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Get Source Metadata"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot - 4 - Query Source"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"oauth"},"2":{"slot":2,"type":"STRING","value":"${oauth}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"sql"},"2":{"slot":2,"type":"STRING","value":"${source_metadata_sql}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"table"},"2":{"slot":2,"type":"STRING","value":"${metadata_stage_table_name}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"warehouse"},"2":{"slot":2,"type":"STRING","value":"${stage_warehouse}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"database"},"2":{"slot":2,"type":"STRING","value":"${stage_database}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"schema"},"2":{"slot":2,"type":"STRING","value":"${stage_schema}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"project"},"2":{"slot":2,"type":"STRING","value":"${stage_project}"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"dataset"},"2":{"slot":2,"type":"STRING","value":"${stage_dataset}"}}},"9":{"slot":9,"values":{"1":{"slot":1,"type":"STRING","value":"s3_bucket_name"},"2":{"slot":2,"type":"STRING","value":"${s3_bucket_name}"}}},"10":{"slot":10,"values":{"1":{"slot":1,"type":"STRING","value":"dist_style"},"2":{"slot":2,"type":"STRING","value":"${dist_style}"}}},"11":{"slot":11,"values":{"1":{"slot":1,"type":"STRING","value":"encryption_method"},"2":{"slot":2,"type":"STRING","value":"${encryption_method}"}}},"12":{"slot":12,"values":{"1":{"slot":1,"type":"STRING","value":"kms_key_id"},"2":{"slot":2,"type":"STRING","value":"${kms_key_id}"}}},"13":{"slot":13,"values":{"1":{"slot":1,"type":"STRING","value":"cloud_storage_area"},"2":{"slot":2,"type":"STRING","value":"${cloud_storage_area}"}}},"14":{"slot":14,"values":{"1":{"slot":1,"type":"STRING","value":"blob_container"},"2":{"slot":2,"type":"STRING","value":"${blob_container}"}}},"15":{"slot":15,"values":{"1":{"slot":1,"type":"STRING","value":"storage_account"},"2":{"slot":2,"type":"STRING","value":"${storage_account}"}}},"16":{"slot":16,"values":{"1":{"slot":1,"type":"STRING","value":"target_table_name"},"2":{"slot":2,"type":"STRING","value":"${target_table_name}"}}},"17":{"slot":17,"values":{"1":{"slot":1,"type":"STRING","value":"autoDebug"},"2":{"slot":2,"type":"STRING","value":"${autoDebug}"}}},"18":{"slot":18,"values":{"1":{"slot":1,"type":"STRING","value":"autoDebugVerbosity"},"2":{"slot":2,"type":"STRING","value":"${autoDebugVerbosity}"}}},"19":{"slot":19,"values":{"1":{"slot":1,"type":"STRING","value":"staging_type"},"2":{"slot":2,"type":"STRING","value":"${staging_type}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"connection_options"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"param"},"2":{"slot":2,"type":"GRID","value":"value"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"connectionOptions"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot Incremental Load"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Incremental"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27126":{"id":27126,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":1010676649,"x":160,"y":0,"width":32,"height":32,"inputConnectorIDs":[27172],"outputSuccessConnectorIDs":[27163],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"metadata_stage_table_name","mapTo":"metadata_stage_table_name","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"Component","mapTo":"component_name","validationStatus":null,"validationMessage":null},"3":{"slot":3,"fromId":null,"fromName":"source_metadata_sql","mapTo":"source_metadata_sql","validationStatus":null,"validationMessage":null},"4":{"slot":4,"fromId":null,"fromName":"get_metadata_sql","mapTo":"get_metadata_sql","validationStatus":null,"validationMessage":null},"5":{"slot":5,"fromId":null,"fromName":"stage_table_name","mapTo":"stage_table_name","validationStatus":null,"validationMessage":null},"6":{"slot":6,"fromId":null,"fromName":"target_table_name","mapTo":"target_table_name","validationStatus":null,"validationMessage":null}},"gridExportMappings":{"1":{"slot":1,"fromGrid":"source_list","toGrid":"source_list","columnMapToFrom":{"isKey":"key_column","isIncremental":"incremental_column","tableName":"table","columnName":"column"},"validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Build Strings"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"stage_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_database}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"stage_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"stage_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_project}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"stage_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"stage_prefix","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_prefix}"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"target_prefix","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_prefix}"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"metadata_stage_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"10":{"slot":10,"name":"stage_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"11":{"slot":11,"name":"target_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"12":{"slot":12,"name":"source_metadata_sql","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"13":{"slot":13,"name":"get_metadata_sql","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"14":{"slot":14,"name":"source_list","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"source_list"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"tableName"},"2":{"slot":2,"type":"GRID","value":"columnName"},"3":{"slot":3,"type":"GRID","value":"isIncremental"},"4":{"slot":4,"type":"GRID","value":"isKey"}}}},"visible":true,"status":{},"columnNames":[]},"15":{"slot":15,"name":"random_suffix","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27127":{"id":27127,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1096987946,"x":640,"y":0,"width":32,"height":32,"inputConnectorIDs":[27165],"outputSuccessConnectorIDs":[27164],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"comp_message","mapTo":"comp_message","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"Component","mapTo":"comp_name","validationStatus":null,"validationMessage":null}},"gridExportMappings":{"1":{"slot":1,"fromGrid":"source_table_metadata","toGrid":"source_table_metadata","columnMapToFrom":{"nullable":"is_nullable","dataTypeName":"data_type_name","isKey":"is_key","length":"length","numericScale":"numeric_scale","ordinal":"ordinal","columnName":"column_name"},"validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Check Source Metadata"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"metadata_stage_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${metadata_stage_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"target_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"get_metadata_sql","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${get_metadata_sql}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"stage_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_database}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"stage_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"stage_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_project}"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"stage_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"source_table_metadata","elements":{},"visible":true,"status":{},"columnNames":[]},"10":{"slot":10,"name":"error_message","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27128":{"id":27128,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-231597779,"x":1040,"y":0,"width":32,"height":32,"inputConnectorIDs":[27157],"outputSuccessConnectorIDs":[27160],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"Component","mapTo":"comp_name","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"error_message","mapTo":"comp_message","validationStatus":null,"validationMessage":null}},"gridExportMappings":{"1":{"slot":1,"fromGrid":"required_columns","toGrid":"source_list","columnMapToFrom":{"isKey":"key_column","isIncremental":"incremental_column","tableName":"table","columnName":"column_name"},"validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Check Required Columns"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"target_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"actual_columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"source_table_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"columnName"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"required_columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"source_list"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"tableName"},"2":{"slot":2,"type":"GRID","value":"columnName"},"3":{"slot":3,"type":"GRID","value":"isIncremental"},"4":{"slot":4,"type":"GRID","value":"isKey"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"missing_columns","elements":{},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"error_message","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27129":{"id":27129,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":947479920,"x":1200,"y":0,"width":32,"height":32,"inputConnectorIDs":[27160],"outputSuccessConnectorIDs":[27158],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"incremental_column","mapTo":"incremental_column","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"incremental_column_count","mapTo":"incremental_column_count","validationStatus":null,"validationMessage":null},"3":{"slot":3,"fromId":null,"fromName":"SELECT_STRING","mapTo":"source_column_list","validationStatus":null,"validationMessage":null},"4":{"slot":4,"fromId":null,"fromName":"key_column_count","mapTo":"key_column_count","validationStatus":null,"validationMessage":null}},"gridExportMappings":{"1":{"slot":1,"fromGrid":"key_columns","toGrid":"key_columns","columnMapToFrom":{"columnName":"column_name"},"validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Parameter Parser"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"platform","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_platform}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"incremental_source_list","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"source_list"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"tableName"},"2":{"slot":2,"type":"GRID","value":"columnName"},"3":{"slot":3,"type":"GRID","value":"isIncremental"},"4":{"slot":4,"type":"GRID","value":"isKey"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"source_list","elements":{},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"stage_warehouse","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_warehouse}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"stage_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_database}"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"stage_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"stage_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_project}"}}}},"visible":true,"status":{},"columnNames":[]},"10":{"slot":10,"name":"stage_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"11":{"slot":11,"name":"stage_prefix","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_prefix}"}}}},"visible":true,"status":{},"columnNames":[]},"12":{"slot":12,"name":"target_warehouse","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_warehouse}"}}}},"visible":true,"status":{},"columnNames":[]},"13":{"slot":13,"name":"target_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"14":{"slot":14,"name":"target_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"15":{"slot":15,"name":"target_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_project}"}}}},"visible":true,"status":{},"columnNames":[]},"16":{"slot":16,"name":"target_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"17":{"slot":17,"name":"target_prefix","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_prefix}"}}}},"visible":true,"status":{},"columnNames":[]},"18":{"slot":18,"name":"maximum_incremental_columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"DECIMAL","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"19":{"slot":19,"name":"source_enclosing_char_e","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"\""}}}},"visible":true,"status":{},"columnNames":[]},"20":{"slot":20,"name":"source_enclosing_char_s","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"\""}}}},"visible":true,"status":{},"columnNames":[]},"21":{"slot":21,"name":"metadata_stage_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${metadata_stage_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"22":{"slot":22,"name":"stage_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"23":{"slot":23,"name":"target_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"24":{"slot":24,"name":"column_count","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"DECIMAL","value":"0"}}}},"visible":true,"status":{},"columnNames":[]},"25":{"slot":25,"name":"key_column_count","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"DECIMAL","value":"0"}}}},"visible":true,"status":{},"columnNames":[]},"26":{"slot":26,"name":"key_columns","elements":{},"visible":true,"status":{},"columnNames":[]},"27":{"slot":27,"name":"incremental_column_count","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"DECIMAL","value":"0"}}}},"visible":true,"status":{},"columnNames":[]},"28":{"slot":28,"name":"incremental_column","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"29":{"slot":29,"name":"SELECT_STRING","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"30":{"slot":30,"name":"stage_table_metadata_sql","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27130":{"id":27130,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":1440,"y":0,"width":32,"height":32,"inputConnectorIDs":[27158],"outputSuccessConnectorIDs":[27156],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Build stage_single_sql"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"# set base stage_single_sql\nstage_single_sql = (f'SELECT {source_column_list} \\n' + \n                    f'FROM \"{table_name}\" \\n'\n                    f'LIMIT 1'\n)\n\ncontext.updateVariable('stage_single_sql', stage_single_sql)\nprint(f\"Updated Variable 'stage_single_sql' with {stage_single_sql}\")\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27131":{"id":27131,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":1600,"y":0,"width":32,"height":32,"inputConnectorIDs":[27156],"outputSuccessConnectorIDs":[27159],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"Component","mapTo":"comp_name","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"error_message","mapTo":"comp_message","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Stage Single Row"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot - 4 - Query Source"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"oauth"},"2":{"slot":2,"type":"STRING","value":"${oauth}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"sql"},"2":{"slot":2,"type":"STRING","value":"${stage_single_sql}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"table"},"2":{"slot":2,"type":"STRING","value":"${stage_table_name}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"warehouse"},"2":{"slot":2,"type":"STRING","value":"${stage_warehouse}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"database"},"2":{"slot":2,"type":"STRING","value":"${stage_database}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"schema"},"2":{"slot":2,"type":"STRING","value":"${stage_schema}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"project"},"2":{"slot":2,"type":"STRING","value":"${stage_project}"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"dataset"},"2":{"slot":2,"type":"STRING","value":"${stage_dataset}"}}},"9":{"slot":9,"values":{"1":{"slot":1,"type":"STRING","value":"s3_bucket_name"},"2":{"slot":2,"type":"STRING","value":"${s3_bucket_name}"}}},"10":{"slot":10,"values":{"1":{"slot":1,"type":"STRING","value":"dist_style"},"2":{"slot":2,"type":"STRING","value":"${dist_style}"}}},"11":{"slot":11,"values":{"1":{"slot":1,"type":"STRING","value":"encryption_method"},"2":{"slot":2,"type":"STRING","value":"${encryption_method}"}}},"12":{"slot":12,"values":{"1":{"slot":1,"type":"STRING","value":"kms_key_id"},"2":{"slot":2,"type":"STRING","value":"${kms_key_id}"}}},"13":{"slot":13,"values":{"1":{"slot":1,"type":"STRING","value":"cloud_storage_area"},"2":{"slot":2,"type":"STRING","value":"${cloud_storage_area}"}}},"14":{"slot":14,"values":{"1":{"slot":1,"type":"STRING","value":"blob_container"},"2":{"slot":2,"type":"STRING","value":"${blob_container}"}}},"15":{"slot":15,"values":{"1":{"slot":1,"type":"STRING","value":"storage_account"},"2":{"slot":2,"type":"STRING","value":"${storage_account}"}}},"16":{"slot":16,"values":{"1":{"slot":1,"type":"STRING","value":"target_table_name"},"2":{"slot":2,"type":"STRING","value":"${target_table_name}"}}},"17":{"slot":17,"values":{"1":{"slot":1,"type":"STRING","value":"autoDebug"},"2":{"slot":2,"type":"STRING","value":"${autoDebug}"}}},"18":{"slot":18,"values":{"1":{"slot":1,"type":"STRING","value":"autoDebugVerbosity"},"2":{"slot":2,"type":"STRING","value":"${autoDebugVerbosity}"}}},"19":{"slot":19,"values":{"1":{"slot":1,"type":"STRING","value":"staging_type"},"2":{"slot":2,"type":"STRING","value":"${staging_type}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"connection_options"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"param"},"2":{"slot":2,"type":"GRID","value":"value"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"connectionOptions"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot Incremental Load"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Incremental"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27132":{"id":27132,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":332733136,"x":1760,"y":0,"width":32,"height":32,"inputConnectorIDs":[27159],"outputSuccessConnectorIDs":[27166],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"load_strategy","mapTo":"load_strategy","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"comp_message","mapTo":"comp_message","validationStatus":null,"validationMessage":null},"3":{"slot":3,"fromId":null,"fromName":"comp_name","mapTo":"comp_name","validationStatus":null,"validationMessage":null},"4":{"slot":4,"fromId":null,"fromName":"stage_sql","mapTo":"stage_sql","validationStatus":null,"validationMessage":null},"5":{"slot":5,"fromId":null,"fromName":"inc_col_type","mapTo":"inc_col_type","validationStatus":null,"validationMessage":null},"6":{"slot":6,"fromId":null,"fromName":"max_value","mapTo":"max_value","validationStatus":null,"validationMessage":null},"7":{"slot":7,"fromId":null,"fromName":"incremental_column","mapTo":"incremental_column","validationStatus":null,"validationMessage":null}},"gridExportMappings":{"1":{"slot":1,"fromGrid":"stage_table_metadata","toGrid":"stage_table_metadata","columnMapToFrom":{"databaseType":"database_type","size":"size","type":"type","precision":"precision","columnName":"column_name"},"validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Decide Load Strategy"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"source_column_list","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${source_column_list}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"key_column_count","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${key_column_count}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"incremental_column_count","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${incremental_column_count}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"incremental_column","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${incremental_column}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"skip_inc_col_check","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"N"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"timestamp_precision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${timestamp_precision}"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"safely_cast_datetime_to_date","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"N"}}}},"visible":true,"status":{},"columnNames":[]},"10":{"slot":10,"name":"encloseTableString","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"TRUE"}}}},"visible":true,"status":{},"columnNames":[]},"11":{"slot":11,"name":"source_enclosing_char_s","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"\""}}}},"visible":true,"status":{},"columnNames":[]},"12":{"slot":12,"name":"source_enclosing_char_e","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"\""}}}},"visible":true,"status":{},"columnNames":[]},"13":{"slot":13,"name":"target_platform","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_platform}"}}}},"visible":true,"status":{},"columnNames":[]},"14":{"slot":14,"name":"stage_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_database}"}}}},"visible":true,"status":{},"columnNames":[]},"15":{"slot":15,"name":"stage_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"16":{"slot":16,"name":"stage_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_project}"}}}},"visible":true,"status":{},"columnNames":[]},"17":{"slot":17,"name":"stage_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"18":{"slot":18,"name":"stage_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"19":{"slot":19,"name":"target_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"20":{"slot":20,"name":"target_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"21":{"slot":21,"name":"target_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_project}"}}}},"visible":true,"status":{},"columnNames":[]},"22":{"slot":22,"name":"target_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"23":{"slot":23,"name":"target_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"24":{"slot":24,"name":"fix_target_data_types","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Yes"}}}},"visible":true,"status":{},"columnNames":[]},"25":{"slot":25,"name":"load_strategy","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"26":{"slot":26,"name":"max_value","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"27":{"slot":27,"name":"stage_sql","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"28":{"slot":28,"name":"inc_col_type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"29":{"slot":29,"name":"stage_table_metadata","elements":{},"visible":true,"status":{},"columnNames":[]},"30":{"slot":30,"name":"messages","elements":{},"visible":true,"status":{},"columnNames":[]},"31":{"slot":31,"name":"use_incremental_column","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Y"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27133":{"id":27133,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":2160,"y":0,"width":32,"height":32,"inputConnectorIDs":[27171],"outputSuccessConnectorIDs":[27162],"outputFailureConnectorIDs":[27174],"outputUnconditionalConnectorIDs":[27173],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"row_count","mapTo":"static_row_count","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"Component","mapTo":"comp_name","validationStatus":null,"validationMessage":null},"3":{"slot":3,"fromId":null,"fromName":"error_message","mapTo":"comp_message","validationStatus":null,"validationMessage":null},"4":{"slot":4,"fromId":null,"fromName":"Status","mapTo":"extract_state","validationStatus":null,"validationMessage":null},"5":{"slot":5,"fromId":null,"fromName":"Duration","mapTo":"stage_duration","validationStatus":null,"validationMessage":null},"6":{"slot":6,"fromId":null,"fromName":"time_taken_to_load","mapTo":"time_taken_to_load","validationStatus":null,"validationMessage":null},"7":{"slot":7,"fromId":null,"fromName":"time_taken_to_stage","mapTo":"time_taken_to_stage","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Stage Data"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot - 4 - Query Source"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"oauth"},"2":{"slot":2,"type":"STRING","value":"${oauth}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"sql"},"2":{"slot":2,"type":"STRING","value":"${stage_sql}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"table"},"2":{"slot":2,"type":"STRING","value":"${stage_table_name}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"warehouse"},"2":{"slot":2,"type":"STRING","value":"${stage_warehouse}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"database"},"2":{"slot":2,"type":"STRING","value":"${stage_database}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"schema"},"2":{"slot":2,"type":"STRING","value":"${stage_schema}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"project"},"2":{"slot":2,"type":"STRING","value":"${stage_project}"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"dataset"},"2":{"slot":2,"type":"STRING","value":"${stage_dataset}"}}},"9":{"slot":9,"values":{"1":{"slot":1,"type":"STRING","value":"s3_bucket_name"},"2":{"slot":2,"type":"STRING","value":"${s3_bucket_name}"}}},"10":{"slot":10,"values":{"1":{"slot":1,"type":"STRING","value":"dist_style"},"2":{"slot":2,"type":"STRING","value":"${dist_style}"}}},"11":{"slot":11,"values":{"1":{"slot":1,"type":"STRING","value":"encryption_method"},"2":{"slot":2,"type":"STRING","value":"${encryption_method}"}}},"12":{"slot":12,"values":{"1":{"slot":1,"type":"STRING","value":"kms_key_id"},"2":{"slot":2,"type":"STRING","value":"${kms_key_id}"}}},"13":{"slot":13,"values":{"1":{"slot":1,"type":"STRING","value":"cloud_storage_area"},"2":{"slot":2,"type":"STRING","value":"${cloud_storage_area}"}}},"14":{"slot":14,"values":{"1":{"slot":1,"type":"STRING","value":"blob_container"},"2":{"slot":2,"type":"STRING","value":"${blob_container}"}}},"15":{"slot":15,"values":{"1":{"slot":1,"type":"STRING","value":"storage_account"},"2":{"slot":2,"type":"STRING","value":"${storage_account}"}}},"16":{"slot":16,"values":{"1":{"slot":1,"type":"STRING","value":"target_table_name"},"2":{"slot":2,"type":"STRING","value":"${target_table_name}"}}},"17":{"slot":17,"values":{"1":{"slot":1,"type":"STRING","value":"autoDebug"},"2":{"slot":2,"type":"STRING","value":"${autoDebug}"}}},"18":{"slot":18,"values":{"1":{"slot":1,"type":"STRING","value":"autoDebugVerbosity"},"2":{"slot":2,"type":"STRING","value":"${autoDebugVerbosity}"}}},"19":{"slot":19,"values":{"1":{"slot":1,"type":"STRING","value":"staging_type"},"2":{"slot":2,"type":"STRING","value":"${staging_type}"}}},"20":{"slot":20,"values":{"1":{"slot":1,"type":"STRING","value":"dataLimit"},"2":{"slot":2,"type":"STRING","value":"${testLimit}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"connection_options"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"param"},"2":{"slot":2,"type":"GRID","value":"value"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"connectionOptions"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"HubSpot Incremental Load"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Incremental"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27134":{"id":27134,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-313802845,"x":2480,"y":0,"width":32,"height":32,"inputConnectorIDs":[27170],"outputSuccessConnectorIDs":[27161],"outputFailureConnectorIDs":[27175],"outputUnconditionalConnectorIDs":[27167],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"Status","mapTo":"load_state","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"comp_message","mapTo":"comp_message","validationStatus":null,"validationMessage":null},"3":{"slot":3,"fromId":null,"fromName":"comp_name","mapTo":"comp_name","validationStatus":null,"validationMessage":null},"4":{"slot":4,"fromId":null,"fromName":"logged_rows_deleted","mapTo":"load_deleted","validationStatus":null,"validationMessage":null},"5":{"slot":5,"fromId":null,"fromName":"logged_rows_inserted","mapTo":"load_inserted","validationStatus":null,"validationMessage":null},"6":{"slot":6,"fromId":null,"fromName":"logged_rows_update","mapTo":"load_updated","validationStatus":null,"validationMessage":null},"7":{"slot":7,"fromId":null,"fromName":"Duration","mapTo":"load_duration","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Load Target"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"platform","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_platform}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"stage_warehouse","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_warehouse}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"stage_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_database}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"stage_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"stage_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_project}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"stage_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"stage_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"target_warehouse","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_warehouse}"}}}},"visible":true,"status":{},"columnNames":[]},"10":{"slot":10,"name":"target_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"11":{"slot":11,"name":"target_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"12":{"slot":12,"name":"target_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_project}"}}}},"visible":true,"status":{},"columnNames":[]},"13":{"slot":13,"name":"target_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"14":{"slot":14,"name":"target_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"15":{"slot":15,"name":"load_strategy","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${load_strategy}"}}}},"visible":true,"status":{},"columnNames":[]},"16":{"slot":16,"name":"dist_style","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${dist_style}"}}}},"visible":true,"status":{},"columnNames":[]},"17":{"slot":17,"name":"columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"stage_table_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"columnName"}}}},"visible":true,"status":{},"columnNames":[]},"18":{"slot":18,"name":"key_columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"key_columns"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"columnName"}}}},"visible":true,"status":{},"columnNames":[]},"19":{"slot":19,"name":"incremental_column","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${incremental_column}"}}}},"visible":true,"status":{},"columnNames":[]},"20":{"slot":20,"name":"join_statement","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"21":{"slot":21,"name":"log_metrics","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"TRUE"}}}},"visible":true,"status":{},"columnNames":[]},"22":{"slot":22,"name":"static_row_count","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${static_row_count}"}}}},"visible":true,"status":{},"columnNames":[]},"23":{"slot":23,"name":"logged_rows_inserted","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"DECIMAL","value":"0"}}}},"visible":true,"status":{},"columnNames":[]},"24":{"slot":24,"name":"logged_rows_update","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"DECIMAL","value":"0"}}}},"visible":true,"status":{},"columnNames":[]},"25":{"slot":25,"name":"logged_rows_deleted","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"DECIMAL","value":"0"}}}},"visible":true,"status":{},"columnNames":[]},"26":{"slot":26,"name":"sub_RangeStartDate","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27135":{"id":27135,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":2640,"y":0,"width":32,"height":32,"inputConnectorIDs":[27161],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Success"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27151":{"id":27151,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":2320,"y":0,"width":32,"height":32,"inputConnectorIDs":[27162],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[27170],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Load Start"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"info"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Load Start"},"2":{"slot":2,"type":"STRING","value":"${load_strategy}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27152":{"id":27152,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":2480,"y":-160,"width":32,"height":32,"inputConnectorIDs":[27167],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Load End"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"info"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Load Inserted Row Count"},"2":{"slot":2,"type":"STRING","value":"${load_inserted}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"Load Updated Row Count"},"2":{"slot":2,"type":"STRING","value":"${load_updated}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"Load Deleted Row Count"},"2":{"slot":2,"type":"STRING","value":"${load_deleted}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"Load Duration"},"2":{"slot":2,"type":"STRING","value":"${load_duration}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"Load End"},"2":{"slot":2,"type":"STRING","value":"${load_state}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27153":{"id":27153,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":2640,"y":160,"width":32,"height":32,"inputConnectorIDs":[27169],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End - Load Failure"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27154":{"id":27154,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":320,"y":0,"width":32,"height":32,"inputConnectorIDs":[27163],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[27168],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Start"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"info"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Iteration Started"},"2":{"slot":2,"type":"STRING","value":"${table_name}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"27155":{"id":27155,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":2480,"y":160,"width":32,"height":32,"inputConnectorIDs":[27175],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[27169],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Load Error"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"error"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Load Error"},"2":{"slot":2,"type":"STRING","value":"${comp_message}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"Load Error At"},"2":{"slot":2,"type":"STRING","value":"${comp_name}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"27156":{"id":27156,"sourceID":27130,"targetID":27131},"27157":{"id":27157,"sourceID":27123,"targetID":27128},"27158":{"id":27158,"sourceID":27129,"targetID":27130},"27159":{"id":27159,"sourceID":27131,"targetID":27132},"27160":{"id":27160,"sourceID":27128,"targetID":27129},"27161":{"id":27161,"sourceID":27134,"targetID":27135},"27162":{"id":27162,"sourceID":27133,"targetID":27151},"27163":{"id":27163,"sourceID":27126,"targetID":27154},"27164":{"id":27164,"sourceID":27127,"targetID":27123},"27165":{"id":27165,"sourceID":27125,"targetID":27127},"27166":{"id":27166,"sourceID":27132,"targetID":27120}},"failureConnectors":{"27174":{"id":27174,"sourceID":27133,"targetID":27122},"27175":{"id":27175,"sourceID":27134,"targetID":27155}},"unconditionalConnectors":{"27167":{"id":27167,"sourceID":27134,"targetID":27152},"27168":{"id":27168,"sourceID":27154,"targetID":27125},"27169":{"id":27169,"sourceID":27155,"targetID":27153},"27170":{"id":27170,"sourceID":27151,"targetID":27134},"27171":{"id":27171,"sourceID":27120,"targetID":27133},"27172":{"id":27172,"sourceID":27124,"targetID":27126},"27173":{"id":27173,"sourceID":27133,"targetID":27121}},"trueConnectors":{},"falseConnectors":{},"iterationConnectors":{},"noteConnectors":{},"canUndo":true,"undoCommand":"Set Parameter","undoCreated":1702919342402,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"27114":{"id":27114,"x":3920,"y":-1342,"width":250,"height":58,"text":"**IGNORE**","colour":"d60000"},"27115":{"id":27115,"x":-2114,"y":644,"width":250,"height":58,"text":"**IGNORE**","colour":"d60000"},"27116":{"id":27116,"x":85,"y":-241,"width":630,"height":318,"text":"** Source Validation **\n\n- (__Build Strings__) Generate generic job variables (stage/target table names etc. - see Export tab) \n\n- (__Get Source Metadata__) Query CData __sys_tablecolumns__ metadata table to get information about the source and store in CDW \n\n- (__Check Source Metadata__) Query table created above, ensure it is valid, export the metadata to a grid (see Grid Export tab)","colour":"e6e63c"},"27117":{"id":27117,"x":804,"y":-237,"width":472,"height":311,"text":"**Input/Target Validation**\n\n- (__Check Required Columns__) Contains logic to validate that the columns which have been requested still exist in the source\n\n- (__Adjust Source List__) Modifies the current __source_list__ grid variable. This will set the correct key and incremental columns (where applicable). \nNB: For HubSpot, the __'Last Modified Date'__,  __'Last modified date'__,  __'CreatedAt'__ column is used as an incremental column when present.\n\n- (__Parameter Parser__) Contains logic to generate some useful Grid/Job variables (e.g key_columns Grid, SELECT list. See Export/Grid Export tabs)","colour":"e6e63c"},"27118":{"id":27118,"x":1366,"y":-235,"width":466,"height":310,"text":"**Decide Load Strategy**\n\n- (__Build stage_single_sql__) Generates and sets stage_single_sql variable which is used to query 1 row from the source. This logic is in its own Python script so that we can log the query.\n\n- (__Stage Single Row__) Executes the __stage_single_sql__ query. This creates a table which will be identical to the stage table for the ETL process, but with only 1 row - the metadata of this table is used within Decide Load Strategy.\n\n- (__Decide Load Strategy__) Executes packaged up logic to determine the type of load this job is doing (UPDATE/APPEND INSERT/TRUNCATE INSERT). \nThis exports the __stage_sql__ variable which will be used to stage the dataset.\nThis exports the __load_strategy__ variable which will be used within Load Target","colour":"e6e63c"},"27119":{"id":27119,"x":1925,"y":-314,"width":631,"height":550,"text":"** Load Data **\n\n- (__Log Extract Start__) Writes metrics to indicate the Extraction phase has started\n\n- (__Stage Data__) Executes __stage_sql__ query and stores the data in the staging table \n\n- (__Load Target__) Executes logic to move the Stage Table Data into the Target Table using the desired __Load Strategy__","colour":"e6e63c"}},"variables":{"oauth":{"definition":{"name":"oauth","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"autoDebug":{"definition":{"name":"autoDebug","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"comp_name":{"definition":{"name":"comp_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"max_value":{"definition":{"name":"max_value","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"stage_sql":{"definition":{"name":"stage_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"testLimit":{"definition":{"name":"testLimit","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"dist_style":{"definition":{"name":"dist_style","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"kms_key_id":{"definition":{"name":"kms_key_id","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"load_state":{"definition":{"name":"load_state","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"table_name":{"definition":{"name":"table_name","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"log_metrics":{"definition":{"name":"log_metrics","type":"TEXT","scope":"BRANCH","description":"TRUE or FALSE (default=TRUE).\nWhether or not to add useful information to the metrics store.\n","visibility":"PUBLIC"},"value":""},"comp_message":{"definition":{"name":"comp_message","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"inc_col_type":{"definition":{"name":"inc_col_type","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"load_deleted":{"definition":{"name":"load_deleted","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"0"},"load_updated":{"definition":{"name":"load_updated","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"0"},"stage_prefix":{"definition":{"name":"stage_prefix","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_schema":{"definition":{"name":"stage_schema","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"staging_type":{"definition":{"name":"staging_type","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"extract_state":{"definition":{"name":"extract_state","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"load_duration":{"definition":{"name":"load_duration","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"load_inserted":{"definition":{"name":"load_inserted","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"0"},"load_strategy":{"definition":{"name":"load_strategy","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"TRUNCATE INSERT"},"stage_dataset":{"definition":{"name":"stage_dataset","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_project":{"definition":{"name":"stage_project","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_prefix":{"definition":{"name":"target_prefix","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_schema":{"definition":{"name":"target_schema","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"blob_container":{"definition":{"name":"blob_container","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"s3_bucket_name":{"definition":{"name":"s3_bucket_name","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_database":{"definition":{"name":"stage_database","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_duration":{"definition":{"name":"stage_duration","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"},"target_dataset":{"definition":{"name":"target_dataset","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_project":{"definition":{"name":"target_project","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_warehouse":{"definition":{"name":"stage_warehouse","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"storage_account":{"definition":{"name":"storage_account","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_database":{"definition":{"name":"target_database","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_platform":{"definition":{"name":"target_platform","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"extract_duration":{"definition":{"name":"extract_duration","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"get_metadata_sql":{"definition":{"name":"get_metadata_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"key_column_count":{"definition":{"name":"key_column_count","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"stage_single_sql":{"definition":{"name":"stage_single_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"stage_table_name":{"definition":{"name":"stage_table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"static_row_count":{"definition":{"name":"static_row_count","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"target_warehouse":{"definition":{"name":"target_warehouse","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"encryption_method":{"definition":{"name":"encryption_method","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_table_name":{"definition":{"name":"target_table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"target_table_name"},"autoDebugVerbosity":{"definition":{"name":"autoDebugVerbosity","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"cloud_storage_area":{"definition":{"name":"cloud_storage_area","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"incremental_column":{"definition":{"name":"incremental_column","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"source_column_list":{"definition":{"name":"source_column_list","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"time_taken_to_load":{"definition":{"name":"time_taken_to_load","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"},"limitedSQLStatement":{"definition":{"name":"limitedSQLStatement","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"source_metadata_sql":{"definition":{"name":"source_metadata_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"time_taken_to_stage":{"definition":{"name":"time_taken_to_stage","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"},"timestamp_precision":{"definition":{"name":"timestamp_precision","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"microseconds"},"incremental_column_count":{"definition":{"name":"incremental_column_count","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"0"},"metadata_stage_table_name":{"definition":{"name":"metadata_stage_table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""}},"grids":{"key_columns":{"definition":{"name":"key_columns","scope":"BRANCH","definitions":[{"name":"columnName","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[]},"source_list":{"definition":{"name":"source_list","scope":"BRANCH","definitions":[{"name":"tableName","type":"TEXT"},{"name":"columnName","type":"TEXT"},{"name":"isIncremental","type":"DECIMAL"},{"name":"isKey","type":"DECIMAL"}],"description":"","visibility":"PUBLIC"},"values":[]},"connection_options":{"definition":{"name":"connection_options","scope":"BRANCH","definitions":[{"name":"param","type":"TEXT"},{"name":"value","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]},"stage_table_metadata":{"definition":{"name":"stage_table_metadata","scope":"BRANCH","definitions":[{"name":"columnName","type":"TEXT"},{"name":"type","type":"TEXT"},{"name":"size","type":"DECIMAL"},{"name":"precision","type":"DECIMAL"},{"name":"databaseType","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[]},"source_table_metadata":{"definition":{"name":"source_table_metadata","scope":"BRANCH","definitions":[{"name":"columnName","type":"TEXT"},{"name":"dataTypeName","type":"TEXT"},{"name":"length","type":"TEXT"},{"name":"numericScale","type":"TEXT"},{"name":"nullable","type":"TEXT"},{"name":"ordinal","type":"TEXT"},{"name":"isKey","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[]}}}},"transformationJobs":{}}],"version":"Built","environment":"bigquery"}