{"objects":[{"metadata":{"identifier":{"packageName":"Matillion Exchange","name":"IBM DB2 Incremental Load","revision":1,"type":"DYNAMIC"},"rootJobReference":{"name":"JDBC Incremental Load - IBM DB2","type":"ORCHESTRATION","parameterMetadata":[{"slot":2,"variableName":"connection_url","variableType":"SCALAR","displayName":"Connection URL","description":"This is the string used to define the connection URL for the source database. This will be passed in from the calling routine.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":3,"variableName":"username","variableType":"SCALAR","displayName":"Username","description":"The username to be supplied to connect to the source.\n\nNote: This applies to the Generic JDBC routine. Different authentication options may be required for the other routines.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":4,"variableName":"password_manager_entry","variableType":"SCALAR","displayName":"Password Manager Entry","description":"The name in the Matillion Password Manager for the password relating to the authentication method.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":5,"variableName":"connectionOptions","variableType":"GRID","displayName":"Connection Options","description":"A list of values and parameters. Parameters and their allowed values are database/driver specific. Referring to the data model will provide insight of what you could provide here.\n\nThey are usually not required as sensible defaults are assumed.","defaultValue":[{"values":{"1":""}}],"defaultValueType":null,"required":true},{"slot":6,"variableName":"source_schema","variableType":"SCALAR","displayName":"Source Schema","description":"The name of the schema in the source database where the tables are located. \n<br>\nThis may not be required for some databases.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":false},{"slot":7,"variableName":"sourceList","variableType":"GRID","displayName":"Tables and Columns","description":"Grid Variable containing, for each column of each table requested:\nTable Name,\nColumn Name,\nWhether the column is Incremental (0 or 1)\nWhether the column is a Key column (0 or 1)","defaultValue":[{"values":{"1":""}}],"defaultValueType":null,"required":true},{"slot":8,"variableName":"bucket","variableType":"SCALAR","displayName":"S3 Bucket Name","description":"<span class=\"env sf\">(AWS Only) </span>The name of an S3 bucket for temporary storage. Ensure your access credentials have S3 access and permission to write to the bucket. See <a href=\"https://redshiftsupport.matillion.com/customer/portal/articles/2054760\" target=\"_blank\">this document</a> for details on setting up access. The temporary objects created in this bucket will be removed again after the load completes, they are not kept.<br>\n<span class=\"env sf\">This property is available when using an Existing Amazon S3 Location for Staging.</span>","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":9,"variableName":"stage_schema","variableType":"SCALAR","displayName":"Stage Schema","description":"The schema name where the staging data will be stored.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":10,"variableName":"stage_prefix","variableType":"SCALAR","displayName":"Stage Prefix","description":"A prefix value that will be added to the start of the stage table names.\n<br>\ne.g. If a Stage Prefix of 'stage_' is specified and the table being processed is named 'test_data' then the target table will be names 'stage_test_data'.","defaultValue":[{"values":{"1":"stage_"}}],"defaultValueType":"TEXT","required":true},{"slot":11,"variableName":"target_schema","variableType":"SCALAR","displayName":"Target Schema","description":"The schema name where the target data will be stored.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":12,"variableName":"target_prefix","variableType":"SCALAR","displayName":"Target Prefix","description":"A prefix value that will be added to the start of the target table names.\n<br>\ne.g. If a Target Prefix of 'target_' is specified and the table being processed is named 'test_data' then the target table will be names 'target_test_data'.","defaultValue":[{"values":{"1":"target_"}}],"defaultValueType":"TEXT","required":false},{"slot":13,"variableName":"dist_style","variableType":"SCALAR","displayName":"Distribution Style","description":"<b>Even:</b> Distributes rows around the Redshift cluster evenly.<br>\n<b>All:</b> Copy rows to all nodes in the Redshift cluster.<br>\nTable distribution is critical to good performance. See the <a href=\"http://docs.aws.amazon.com/redshift/latest/dg/tutorial-tuning-tables-distribution.html\" target=\"_blank\">Amazon Redshift</a> documentation for more information.","defaultValue":[{"values":{"1":"Even"}}],"defaultValueType":"TEXT","required":true},{"slot":14,"variableName":"load_concurrent","variableType":"SCALAR","displayName":"Load Type","description":"Sequential - Iterations are done in sequence, waiting for each to complete before starting the next. \nThis is the default.\n<br>\nConcurrent - Iterations are run concurrently. This requires all \"Variables to Iterate\" to be defined as\n<a href=\"https://redshiftsupport.matillion.com/customer/portal/articles/2037630?b_id=8915#header1\" target=\"_blank\"><i>Copied</i> variables</a>, so that each iteration gets its own copy of the variable isolated from the same\nvariable being used by other concurrent executions.\n<span class=\"env rs sf\"><br><b>Note:</b> The maximum concurrency is limited by the number of available threads (2x the number of virtual cpus on your cloud instance).</span>","defaultValue":[{"values":{"1":"Sequential"}}],"defaultValueType":"TEXT","required":true},{"slot":15,"variableName":"encryption_method","variableType":"SCALAR","displayName":"Encryption","description":"<span class=\"env sf\">(AWS Only) </span>Decide on how the files are encrypted inside the S3 Bucket.<span class=\"env sf\">This property is available when using an Existing Amazon S3 Location for Staging.</span><br>\n                <b>None:</b> No encryption.<br>\n                <b>SSE KMS:</b> Encrypt the data according to a key stored on <a href=\"https://aws.amazon.com/kms/\" target=\"_blank\">KMS</a>.<br>\n                <b>SSE S3:</b> Encrypt the data according to a key stored on an <a href=\"http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html\" target=\"_blank\">S3 bucket</a>","defaultValue":[{"values":{"1":"None"}}],"defaultValueType":"TEXT","required":true},{"slot":16,"variableName":"kms_key_id","variableType":"SCALAR","displayName":"KMS Key ID","description":"The ID of the KMS encryption key you have chosen to use in the 'Encryption' property.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":false},{"slot":17,"variableName":"log_metrics","variableType":"SCALAR","displayName":"Log Metrics","description":"TRUE or FALSE (default=TRUE).\nWhether or not to add useful information to the metrics store.\n","defaultValue":[{"values":{"1":"TRUE"}}],"defaultValueType":"TEXT","required":true}]},"orchestrationJobs":["JDBC Incremental Load - IBM DB2"],"transformationJobs":[],"imageMetadata":{"componentIcon":"iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAEiUlEQVR42u2VCUxcVRSGAWNs1SCCWCxGIUiIaZDEhLUhjYGBYWshI0urlN2BADPDMswAw8CwDEvYK8NqiLGJQNsQm6it2qKFVDEERRpD3ahLa5ugLWq1ivD7zgGeBWrKICE18SZ/5p5lzv3evee+Z2Hx//h7OAtK3CI53wogSRC2SIl3PkBQUBDCwsJYgYGBcHNzE21SSEgIHBwcxNzQ0FCWra0t++zs7DiPfAEBAeYDFBUVwWAwsGhORcrLy0UfSa1Ww9HREYWFhaLP19eX/+/v7y/6cnNzzQeQyWS8cHFxMc89PDyQmZmJiooKpKamIj4+noEiIiIQHR3NeWTL5XJYWloiPT2dbaoRFRW1sR5QKBRQqVSi7enpyQDe3t58JDSXSqUcIzhakHwuLi6orKzkeVZW1sabMDs7myGWbS8vrxXHoNPpYG1tzbGMjAyUlZXxEyuVSuTk5KCgoIB3YsMAdHb5+fmi7ePjg5qaGlRXV7OMRiPs7e3F3crLy0NycjLH0tLSGIBgNgxABbRarWj7+fmhsbERrq6usLGx4XlcXJwIq9FouFfIT8dFu0FQGwKgrW5oaGDRk9J1osKrJZFIoNfrOY8AqAFLS0thZWXFAOSnWmYDUBM1NzezamtrER4eLtokKhwTE8O51A/kowXJdnJy4l/qEfJXVVX9B9+EWw+Q4J5kcdAdm627Dj611h/vfguAlsAkixYJNlP3N4UivDxtbaxVcnuAhzpl8B1Qwqdfwb+P9z4vxh7piYWf4PMbUHH80ZcOrFhge1s4HmuPxQslOZALcq6T4Z5mqXkAyW/XY/WoHz/CsYKR7hX+BUEN40c5tv+EEdfnbmBkZIQ/UvSN0BRpoS7SYE9lIqxagtYHkLQEUDF6mJ9y6sdv2H7ylRQRQHe2F3uO5uHqjZ8xN/8ntrWFYea3WXx//Qe4vpyIrPYymEwm7O5TYldNLFTFauw1yM0DyBvuZFv3fi/bCW/ViQCp7zRybGLmK/w+P4cH2iPRd6QfHaOD7G8UdmVychJ7j+vZ3lG/DwXCbuzXZ8vNBsg50842Lb4M8OHl8zhz8ROeq4e7uOlo23fWR8KmIwqfX7uIPwSwnT1xYt19wg4IOf1mAyjfM7GtGekRASZnpjH07ceYX1jAya/H8ERjDAM83Pksxq58xjn5AtjNdb2N8ZQzbjYAFaKRdqpJBEhZOoJXzw+x/dxAGQOMXjjHjUnQq6/gM5VJlHPabIDDU6fY9h1QrOmB16dH2Q4e1EKpU2NoaAjlQvNSbFuTdLHzl+rKDBlQF2p06wa48utVTM9eXjrzKVi1BokA3/0ygws/Lca+nL2E+0wRODl8GiUlJTjxwbv44tolnP10HFGGdBGC3gd2DRG3vwX0ouk59ya6J9+AaeI4N5lNR+TiNh5To+WjQTSNH0PdWL8Q68aO7mjcfUiKFydeQ0pbMbTCUXR1daGvr4+P5WaIdV3Dfyu6cpKKFCToFQyQWqLC9qaQrQNYlqNwJWnxe5cX/0cAU9iDFq3BXputp2sOBO8yxuxe4ae17pTxFzepzvdeGThvAAAAAElFTkSuQmCC","smallIcon":"iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAB/ElEQVR42q2Q3WuSYRjG7cN/QEoFNQgVdtI6Er9AWNC2gxAkKiO2IC2Yb5kfQwfKcmQMc+mgeRCtGrFBQg01KGOQBMuKxYaEo4MaGwXVQR0EHf9637cCl8gadMMF9/NwXT/u+1Yo/lONiCptUyOtAOkDk8kkS6/XYzab0Wq1qFQquVcqlWg0GoxGI5L3d2YzIBKJEI1G8Xq9JBIJYrEYbrebVCpFV1cXgUCAUCjUGWC1WrHZbOh0OtkoBdRqtQzy+/0kk0ksFktngMPhwOl0YjAYCIfDCIIgw6SpJJjP58Nut3cGZDIZstkswWCQfD4vj+7xeEin0/IKLpdL9nQEbFMtgLi9JIqttCvuYE+058+7BZDvLSkmD3O+NsV08zFC7TqmmdPcalYpNCoYZwY5/ihNaGqMwdELHBw/yQ4p8zeg9L7OkXKS5tcNBp5kqK4vkXp5l8uvZnn+dpn+uWGS5QIHxr0cTQ1V2gBlEeAoXqS6sUR88Saff3xj/fsXvKUx+q+cZf7dImcWJpC83lFhrSOg/mkV38I1KmsvOCGOfm/1KdNzMyTrd9g52SsDTo0KK22A3PIDah8aTDXKdM+e49nHN+Iar+m5P0y2eIPiw3lytwvsnXCzO9/XfoOtpBGDxy4Nse+qB8WmI+b6VIrcId2/qDs7sP9XL2bE+gmbVXlv8DXnMwAAAABJRU5ErkJggg=="},"description":"","helpHtml":"<div class=\"mtln\"><img src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAEiUlEQVR42u2VCUxcVRSGAWNs1SCCWCxGIUiIaZDEhLUhjYGBYWshI0urlN2BADPDMswAw8CwDEvYK8NqiLGJQNsQm6it2qKFVDEERRpD3ahLa5ugLWq1ivD7zgGeBWrKICE18SZ/5p5lzv3evee+Z2Hx//h7OAtK3CI53wogSRC2SIl3PkBQUBDCwsJYgYGBcHNzE21SSEgIHBwcxNzQ0FCWra0t++zs7DiPfAEBAeYDFBUVwWAwsGhORcrLy0UfSa1Ww9HREYWFhaLP19eX/+/v7y/6cnNzzQeQyWS8cHFxMc89PDyQmZmJiooKpKamIj4+noEiIiIQHR3NeWTL5XJYWloiPT2dbaoRFRW1sR5QKBRQqVSi7enpyQDe3t58JDSXSqUcIzhakHwuLi6orKzkeVZW1sabMDs7myGWbS8vrxXHoNPpYG1tzbGMjAyUlZXxEyuVSuTk5KCgoIB3YsMAdHb5+fmi7ePjg5qaGlRXV7OMRiPs7e3F3crLy0NycjLH0tLSGIBgNgxABbRarWj7+fmhsbERrq6usLGx4XlcXJwIq9FouFfIT8dFu0FQGwKgrW5oaGDRk9J1osKrJZFIoNfrOY8AqAFLS0thZWXFAOSnWmYDUBM1NzezamtrER4eLtokKhwTE8O51A/kowXJdnJy4l/qEfJXVVX9B9+EWw+Q4J5kcdAdm627Dj611h/vfguAlsAkixYJNlP3N4UivDxtbaxVcnuAhzpl8B1Qwqdfwb+P9z4vxh7piYWf4PMbUHH80ZcOrFhge1s4HmuPxQslOZALcq6T4Z5mqXkAyW/XY/WoHz/CsYKR7hX+BUEN40c5tv+EEdfnbmBkZIQ/UvSN0BRpoS7SYE9lIqxagtYHkLQEUDF6mJ9y6sdv2H7ylRQRQHe2F3uO5uHqjZ8xN/8ntrWFYea3WXx//Qe4vpyIrPYymEwm7O5TYldNLFTFauw1yM0DyBvuZFv3fi/bCW/ViQCp7zRybGLmK/w+P4cH2iPRd6QfHaOD7G8UdmVychJ7j+vZ3lG/DwXCbuzXZ8vNBsg50842Lb4M8OHl8zhz8ROeq4e7uOlo23fWR8KmIwqfX7uIPwSwnT1xYt19wg4IOf1mAyjfM7GtGekRASZnpjH07ceYX1jAya/H8ERjDAM83Pksxq58xjn5AtjNdb2N8ZQzbjYAFaKRdqpJBEhZOoJXzw+x/dxAGQOMXjjHjUnQq6/gM5VJlHPabIDDU6fY9h1QrOmB16dH2Q4e1EKpU2NoaAjlQvNSbFuTdLHzl+rKDBlQF2p06wa48utVTM9eXjrzKVi1BokA3/0ygws/Lca+nL2E+0wRODl8GiUlJTjxwbv44tolnP10HFGGdBGC3gd2DRG3vwX0ouk59ya6J9+AaeI4N5lNR+TiNh5To+WjQTSNH0PdWL8Q68aO7mjcfUiKFydeQ0pbMbTCUXR1daGvr4+P5WaIdV3Dfyu6cpKKFCToFQyQWqLC9qaQrQNYlqNwJWnxe5cX/0cAU9iDFq3BXputp2sOBO8yxuxe4ae17pTxFzepzvdeGThvAAAAAElFTkSuQmCC\"><h3>IBM DB2 Incremental Load</h3><p style='white-space: pre-line; word-wrap: break-word;'></p><h3>Properties</h3><table><tbody><tr><th>Property</th><th>Type</th><th>Description</th></tr><tr><td>Connection URL</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>This is the string used to define the connection URL for the source database. This will be passed in from the calling routine.</td></tr><tr><td>Username</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>The username to be supplied to connect to the source.\n\nNote: This applies to the Generic JDBC routine. Different authentication options may be required for the other routines.</td></tr><tr><td>Password Manager Entry</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>The name in the Matillion Password Manager for the password relating to the authentication method.</td></tr><tr><td>Connection Options</td><td>Grid</td><td style='white-space: pre-line; word-wrap: break-word;'>A list of values and parameters. Parameters and their allowed values are database/driver specific. Referring to the data model will provide insight of what you could provide here.\n\nThey are usually not required as sensible defaults are assumed.</td></tr><tr><td>Source Schema</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>The name of the schema in the source database where the tables are located. \n<br>\nThis may not be required for some databases.</td></tr><tr><td>Tables and Columns</td><td>Grid</td><td style='white-space: pre-line; word-wrap: break-word;'>Grid Variable containing, for each column of each table requested:\nTable Name,\nColumn Name,\nWhether the column is Incremental (0 or 1)\nWhether the column is a Key column (0 or 1)</td></tr><tr><td>S3 Bucket Name</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'><span class=\"env sf\">(AWS Only) </span>The name of an S3 bucket for temporary storage. Ensure your access credentials have S3 access and permission to write to the bucket. See <a href=\"https://redshiftsupport.matillion.com/customer/portal/articles/2054760\" target=\"_blank\">this document</a> for details on setting up access. The temporary objects created in this bucket will be removed again after the load completes, they are not kept.<br>\n<span class=\"env sf\">This property is available when using an Existing Amazon S3 Location for Staging.</span></td></tr><tr><td>Stage Schema</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>The schema name where the staging data will be stored.</td></tr><tr><td>Stage Prefix</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>A prefix value that will be added to the start of the stage table names.\n<br>\ne.g. If a Stage Prefix of 'stage_' is specified and the table being processed is named 'test_data' then the target table will be names 'stage_test_data'.</td></tr><tr><td>Target Schema</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>The schema name where the target data will be stored.</td></tr><tr><td>Target Prefix</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>A prefix value that will be added to the start of the target table names.\n<br>\ne.g. If a Target Prefix of 'target_' is specified and the table being processed is named 'test_data' then the target table will be names 'target_test_data'.</td></tr><tr><td>Distribution Style</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'><b>Even:</b> Distributes rows around the Redshift cluster evenly.<br>\n<b>All:</b> Copy rows to all nodes in the Redshift cluster.<br>\nTable distribution is critical to good performance. See the <a href=\"http://docs.aws.amazon.com/redshift/latest/dg/tutorial-tuning-tables-distribution.html\" target=\"_blank\">Amazon Redshift</a> documentation for more information.</td></tr><tr><td>Load Type</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>Sequential - Iterations are done in sequence, waiting for each to complete before starting the next. \nThis is the default.\n<br>\nConcurrent - Iterations are run concurrently. This requires all \"Variables to Iterate\" to be defined as\n<a href=\"https://redshiftsupport.matillion.com/customer/portal/articles/2037630?b_id=8915#header1\" target=\"_blank\"><i>Copied</i> variables</a>, so that each iteration gets its own copy of the variable isolated from the same\nvariable being used by other concurrent executions.\n<span class=\"env rs sf\"><br><b>Note:</b> The maximum concurrency is limited by the number of available threads (2x the number of virtual cpus on your cloud instance).</span></td></tr><tr><td>Encryption</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'><span class=\"env sf\">(AWS Only) </span>Decide on how the files are encrypted inside the S3 Bucket.<span class=\"env sf\">This property is available when using an Existing Amazon S3 Location for Staging.</span><br>\n                <b>None:</b> No encryption.<br>\n                <b>SSE KMS:</b> Encrypt the data according to a key stored on <a href=\"https://aws.amazon.com/kms/\" target=\"_blank\">KMS</a>.<br>\n                <b>SSE S3:</b> Encrypt the data according to a key stored on an <a href=\"http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html\" target=\"_blank\">S3 bucket</a></td></tr><tr><td>KMS Key ID</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>The ID of the KMS encryption key you have chosen to use in the 'Encryption' property.</td></tr><tr><td>Log Metrics</td><td>Variable</td><td style='white-space: pre-line; word-wrap: break-word;'>TRUE or FALSE (default=TRUE).\nWhether or not to add useful information to the metrics store.\n</td></tr></tbody></table></div>","created":1608281191256},"orchestrationJobs":{"JDBC Incremental Load - IBM DB2":{"id":-2,"revision":3,"created":1608280785325,"timestamp":1608280785325,"components":{"3922":{"id":3922,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":0,"y":0,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[3927],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"3923":{"id":3923,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":323,"y":76,"width":32,"height":32,"inputConnectorIDs":[3928],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Failure"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"3924":{"id":3924,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":323,"y":-80,"width":32,"height":32,"inputConnectorIDs":[3926],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Success"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"3925":{"id":3925,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":1770048173,"x":160,"y":0,"width":32,"height":32,"inputConnectorIDs":[3927],"outputSuccessConnectorIDs":[3926],"outputFailureConnectorIDs":[3928],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"JDBC Incremental Load"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Database Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${database_type}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Connection URL","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${connection_url}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Username","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${username}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Password Manager Entry","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${password_manager_entry}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Source Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${source_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${load_concurrent}"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Table and Columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"sourceList"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"tableName"},"2":{"slot":2,"type":"GRID","value":"columnName"},"3":{"slot":3,"type":"GRID","value":"isIncremental"},"4":{"slot":4,"type":"GRID","value":"isKey"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Connection Options","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"connectionOptions"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"parameter"},"2":{"slot":2,"type":"GRID","value":"value"}}}},"visible":true,"status":{},"columnNames":[]},"10":{"slot":10,"name":"Distribution Style","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${dist_style}"}}}},"visible":true,"status":{},"columnNames":[]},"11":{"slot":11,"name":"S3 Bucket Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${bucket}"}}}},"visible":true,"status":{},"columnNames":[]},"12":{"slot":12,"name":"Stage Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"13":{"slot":13,"name":"Stage Prefix","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_prefix}"}}}},"visible":true,"status":{},"columnNames":[]},"14":{"slot":14,"name":"Target Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"15":{"slot":15,"name":"Target Prefix","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_prefix}"}}}},"visible":true,"status":{},"columnNames":[]},"16":{"slot":16,"name":"Encryption","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${encryption_method}"}}}},"visible":true,"status":{},"columnNames":[]},"17":{"slot":17,"name":"KMS Key ID","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${kms_key_id}"}}}},"visible":true,"status":{},"columnNames":[]},"18":{"slot":18,"name":"Log Metrics","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"TRUE"}}}},"visible":true,"status":{},"columnNames":[]},"19":{"slot":19,"name":"Automatically Update Target Metadata","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Yes"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"3926":{"id":3926,"sourceID":3925,"targetID":3924}},"failureConnectors":{"3928":{"id":3928,"sourceID":3925,"targetID":3923}},"unconditionalConnectors":{"3927":{"id":3927,"sourceID":3922,"targetID":3925}},"trueConnectors":{},"falseConnectors":{},"iterationConnectors":{},"noteConnectors":{},"canUndo":true,"undoCommand":"Move Components","undoCreated":1608280729923,"canRedo":false,"redoCommand":"","redoCreated":-1,"notes":{"3921":{"id":3921,"x":-474,"y":-234,"width":391,"height":419,"text":"** Generic JDBC Pipeline Template **\n\nThis is a generic template to create a JDBC Incremental Load Shared Job using a new JDBC Driver. This is done by adjusting some of the Private Job/Grid Variables to get the correct behaviour for the specific Database. \nSteps:\n\n**1.** Replace the __database_type__ private variable with the required database type, this is what is selected in the Database Query component and the same as the name in the jdbc-providers.properties file. You will also need to set the __source_schema__ variable to private if it not required for the particular flavour.\n\n**2.** Fill in the __sqlFlavourConfig__ Grid Variable - you will need to supply parameters and values for __limit_type, enclosing_character_start, enclosing_character_end__ which will be used by the JDBC Incremental Load Job.\nFor example:\nlimit_type\ttop-n\nenclosing_character_start\t\"\nenclosing_character_end\t\"\n\n**3.** Rename this job to include the __database_type__, e.g. \"JDBC Incremental Load - Greenplum\"\n\n**4.** In the Test folder, update both the \"Test - Smoke\" and \"Test - Incremental Load\" jobs with the connection and source details.  Ensure both tests succeed before generating a Shared Job.","colour":"00ce4f"}},"variables":{"bucket":{"definition":{"name":"bucket","type":"TEXT","scope":"BRANCH","description":"<span class=\"env sf\">(AWS Only) </span>The name of an S3 bucket for temporary storage. Ensure your access credentials have S3 access and permission to write to the bucket. See <a href=\"https://redshiftsupport.matillion.com/customer/portal/articles/2054760\" target=\"_blank\">this document</a> for details on setting up access. The temporary objects created in this bucket will be removed again after the load completes, they are not kept.<br>\n<span class=\"env sf\">This property is available when using an Existing Amazon S3 Location for Staging.</span>","visibility":"PUBLIC"},"value":null},"username":{"definition":{"name":"username","type":"TEXT","scope":"BRANCH","description":"The username to be supplied to connect to the source.\n\nNote: This applies to the Generic JDBC routine. Different authentication options may be required for the other routines.","visibility":"PUBLIC"},"value":null},"dist_style":{"definition":{"name":"dist_style","type":"TEXT","scope":"BRANCH","description":"<b>Even:</b> Distributes rows around the Redshift cluster evenly.<br>\n<b>All:</b> Copy rows to all nodes in the Redshift cluster.<br>\nTable distribution is critical to good performance. See the <a href=\"http://docs.aws.amazon.com/redshift/latest/dg/tutorial-tuning-tables-distribution.html\" target=\"_blank\">Amazon Redshift</a> documentation for more information.","visibility":"PUBLIC"},"value":"Even"},"kms_key_id":{"definition":{"name":"kms_key_id","type":"TEXT","scope":"BRANCH","description":"The ID of the KMS encryption key you have chosen to use in the 'Encryption' property.","visibility":"PUBLIC"},"value":""},"log_metrics":{"definition":{"name":"log_metrics","type":"TEXT","scope":"BRANCH","description":"TRUE or FALSE (default=TRUE).\nWhether or not to add useful information to the metrics store.\n","visibility":"PUBLIC"},"value":"TRUE"},"stage_prefix":{"definition":{"name":"stage_prefix","type":"TEXT","scope":"BRANCH","description":"A prefix value that will be added to the start of the stage table names.\n\ne.g. If a Stage Prefix of 'stage_' is specified and the table being processed is named 'test_data' then the target table will be named 'stage_test_data'.","visibility":"PUBLIC"},"value":"stage_"},"stage_schema":{"definition":{"name":"stage_schema","type":"TEXT","scope":"BRANCH","description":"The schema name where the staging data will be stored.","visibility":"PUBLIC"},"value":null},"database_type":{"definition":{"name":"database_type","type":"TEXT","scope":"BRANCH","description":"The type of the JDBC database to connect to.\n<br>\n<b>Microsoft SQL Server:</b> see <a target=\"_blank\" href=\"http://www.microsoft.com/\">their website</a> for more details.\n<br>\n<b>Oracle:</b> see <a target=\"_blank\" href=\"http://www.oracle.com/index.html\">their website</a> for more details.\n<br>\n<b>PostgreSQL:</b> see <a target=\"_blank\" href=\"http://www.postgresql.org/\">their website</a> for more details.\n<br>\n<b>MySQL:</b> see <a target=\"_blank\" href=\"https://www.mysql.com/\">their website</a> for more details.\n<br>\n<b>IBM DB2:</b> see <a target=\"_blank\" href=\"http://www-01.ibm.com/software/data/db2/\">their website</a> for more details.\n<br>\n<b>Note:</b> For some databases, you must first provide a JDBC driver as \nnot all drivers can be distributed with Matillion ETL. See\n<a target=\"_blank\" href=\"https://redshiftsupport.matillion.com/customer/portal/articles/2406136\">this article</a> \nfor instructions on managing drivers.   ","visibility":"PRIVATE"},"value":"IBM DB2"},"source_schema":{"definition":{"name":"source_schema","type":"TEXT","scope":"BRANCH","description":"The name of the schema in the source database where the tables are located.\nThis may not be required for some databases.","visibility":"PUBLIC"},"value":null},"target_prefix":{"definition":{"name":"target_prefix","type":"TEXT","scope":"BRANCH","description":"A prefix value that will be added to the start of the target table names.\n\ne.g. If a Target Prefix of 'target_' is specified and the table being processed is named 'test_data' then the target table will be named 'target_test_data'.","visibility":"PUBLIC"},"value":"stage_"},"target_schema":{"definition":{"name":"target_schema","type":"TEXT","scope":"BRANCH","description":"The schema name where the target data will be stored.","visibility":"PUBLIC"},"value":null},"connection_url":{"definition":{"name":"connection_url","type":"TEXT","scope":"BRANCH","description":"This is the string used to define the connection URL for the source database. This will be passed in from the calling routine.","visibility":"PUBLIC"},"value":null},"load_concurrent":{"definition":{"name":"load_concurrent","type":"TEXT","scope":"BRANCH","description":"Sequential - Iterations are done in sequence, waiting for each to complete before starting the next. \nThis is the default.\n<br>\nConcurrent - Iterations are run concurrently. This requires all \"Variables to Iterate\" to be defined as\n<a href=\"https://redshiftsupport.matillion.com/customer/portal/articles/2037630?b_id=8915#header1\" target=\"_blank\"><i>Copied</i> variables</a>, so that each iteration gets its own copy of the variable isolated from the same\nvariable being used by other concurrent executions.\n<span class=\"env rs sf\"><br><b>Note:</b> The maximum concurrency is limited by the number of available threads (2x the number of virtual cpus on your cloud instance).</span>","visibility":"PUBLIC"},"value":"Sequential"},"encryption_method":{"definition":{"name":"encryption_method","type":"TEXT","scope":"BRANCH","description":"<span class=\"env sf\">(AWS Only) </span>Decide on how the files are encrypted inside the S3 Bucket.<span class=\"env sf\">This property is available when using an Existing Amazon S3 Location for Staging.</span><br>\n                <b>None:</b> No encryption.<br>\n                <b>SSE KMS:</b> Encrypt the data according to a key stored on <a href=\"https://aws.amazon.com/kms/\" target=\"_blank\">KMS</a>.<br>\n                <b>SSE S3:</b> Encrypt the data according to a key stored on an <a href=\"http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html\" target=\"_blank\">S3 bucket</a>","visibility":"PUBLIC"},"value":"None"},"password_manager_entry":{"definition":{"name":"password_manager_entry","type":"TEXT","scope":"BRANCH","description":"The name in the Matillion Password Manager for the password relating to the authentication method.","visibility":"PUBLIC"},"value":null}},"grids":{"sourceList":{"definition":{"name":"sourceList","scope":"BRANCH","definitions":[{"name":"tableName","type":"TEXT"},{"name":"columnName","type":"TEXT"},{"name":"isIncremental","type":"DECIMAL"},{"name":"isKey","type":"DECIMAL"}],"description":"Grid Variable containing, for each column of each table requested:\nTable Name,\nColumn Name,\nWhether the column is Incremental (0 or 1)\nWhether the column is a Key column (0 or 1)","visibility":"PUBLIC"},"values":[]},"sqlFlavourConfig":{"definition":{"name":"sqlFlavourConfig","scope":"BRANCH","definitions":[{"name":"parameter","type":"TEXT"},{"name":"value","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[{"values":["limit_type","limit-outer"]},{"values":["enclosing_character_start","\""]},{"values":["enclosing_character_end","\""]}]},"connectionOptions":{"definition":{"name":"connectionOptions","scope":"BRANCH","definitions":[{"name":"parameter","type":"TEXT"},{"name":"value","type":"TEXT"}],"description":"A list of values and parameters. Parameters and their allowed values are database/driver specific. Referring to the data model will provide insight of what you could provide here.\n\nThey are usually not required as sensible defaults are assumed.","visibility":"PUBLIC"},"values":[]}}}},"transformationJobs":{}}],"version":"EMD_19473_jdbc_new","environment":"redshift"}