{"objects":[{"metadata":{"identifier":{"packageName":"Matillion Exchange","name":"Facebook Content Insights Incremental Load","revision":2,"type":"DYNAMIC"},"rootJobReference":{"name":"Facebook Content Insights - 1 - Orchestrator","type":"ORCHESTRATION","parameterMetadata":[{"slot":2,"variableName":"oauth","variableType":"SCALAR","displayName":"OAuth","description":"The name of the OAuth entry to be used from the Matillion OAuth Manager to connect to the source. These must be set up in advance, using Project → Manage OAuth.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":3,"variableName":"key_value","variableType":"SCALAR","displayName":"Target Value","description":"This is the Id of the page or post to get Content Insights for.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":4,"variableName":"source_list","variableType":"GRID","displayName":"Tables and Columns","description":"Contains the list of tables and columns (and in some cases an incremental_column to specify which column the load should be incremented on [accepts values of 0 or 1]) to be processed. ","defaultValue":[{"values":{"1":""}}],"defaultValueType":null,"required":true},{"slot":5,"variableName":"advanced_connection_options","variableType":"GRID","displayName":"Connection Options","description":"A list of values and parameters. Parameters and their allowed values are database/driver specific. Referring to the data model will provide insight of what you could provide here.\n
\nThey are usually not required as sensible defaults are assumed.","defaultValue":[{"values":{"1":""}}],"defaultValueType":null,"required":true},{"slot":6,"variableName":"initial_start_date","variableType":"SCALAR","displayName":"Initial Start Date","description":"","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":false},{"slot":7,"variableName":"period","variableType":"SCALAR","displayName":"Period","description":"The period for the insight. This must always be specified.\n
\nThe allowed values are day, week, days_28, lifetime.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":8,"variableName":"instagram_business_account_id","variableType":"SCALAR","displayName":"Instagram Business Account Id","description":"Instagram Business Account Id. \n
\nTarget for Instagram Content Insights. ","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":false},{"slot":9,"variableName":"load_concurrent","variableType":"SCALAR","displayName":"Load Type","description":"Sequential - Iterations are done in sequence, waiting for each to complete before starting the next. \nThis is the default.\n
\nConcurrent - Iterations are run concurrently. This requires all \"Variables to Iterate\" to be defined as\nCopied variables, so that each iteration gets its own copy of the variable isolated from the same\nvariable being used by other concurrent executions.\n
Note: The maximum concurrency is limited by the number of available threads (2x the number of virtual cpus on your cloud instance).","defaultValue":[{"values":{"1":"Sequential"}}],"defaultValueType":"TEXT","required":true},{"slot":10,"variableName":"dist_style","variableType":"SCALAR","displayName":"Distribution Style","description":"Even: Distributes rows around the Redshift cluster evenly.
\nAll: Copy rows to all nodes in the Redshift cluster.
\nKey: Distribute rows around the Redshift cluster according to the value of a key column.
\nTable distribution is critical to good performance. See the Amazon Redshift documentation for more information.","defaultValue":[{"values":{"1":"Auto"}}],"defaultValueType":"TEXT","required":true},{"slot":11,"variableName":"s3_bucket_name","variableType":"SCALAR","displayName":"S3 Bucket Name","description":"(AWS Only) The name of an S3 bucket for temporary storage. Ensure your access credentials have S3 access and permission to write to the bucket. See this document for details on setting up access. The temporary objects created in this bucket will be removed again after the load completes, they are not kept.
\nThis property is available when using an Existing Amazon S3 Location for Staging.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":12,"variableName":"stage_prefix","variableType":"SCALAR","displayName":"Stage Prefix","description":"A prefix value that will be added to the start of the stage table names.\n
\ne.g. If a Sarget Prefix of 'stage_' is specified and the table being processed is named 'test_data' then the target table will be names 'stage_test_data'.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":13,"variableName":"stage_schema","variableType":"SCALAR","displayName":"Stage Schema","description":"The schema name where the staging data will be stored.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":14,"variableName":"target_prefix","variableType":"SCALAR","displayName":"Target Prefix","description":"A prefix value that will be added to the start of the target table names.\n
\ne.g. If a Target Prefix of 'target_' is specified and the table being processed is named 'test_data' then the target table will be names 'target_test_data'.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":false},{"slot":15,"variableName":"target_schema","variableType":"SCALAR","displayName":"Target Schema","description":"The schema name where the target data will be stored.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":true},{"slot":16,"variableName":"encryption_method","variableType":"SCALAR","displayName":"Encryption","description":"(AWS Only) Decide on how the files are encrypted inside the S3 Bucket.This property is available when using an Existing Amazon S3 Location for Staging.
\n None: No encryption.
\n SSE KMS: Encrypt the data according to a key stored on KMS.
\n SSE S3: Encrypt the data according to a key stored on an S3 bucket","defaultValue":[{"values":{"1":"None"}}],"defaultValueType":"TEXT","required":false},{"slot":17,"variableName":"kms_key_id","variableType":"SCALAR","displayName":"KMS Key Id","description":"The ID of the KMS encryption key you have chosen to use in the 'Encryption' property.","defaultValue":[{"values":{"1":""}}],"defaultValueType":"TEXT","required":false},{"slot":18,"variableName":"log_metrics","variableType":"SCALAR","displayName":"Log Metrics","description":"Option to switch off Metrics logging in the full product - default is TRUE which records the metrics for the batch run. ","defaultValue":[{"values":{"1":"TRUE"}}],"defaultValueType":"TEXT","required":false}]},"orchestrationJobs":["Facebook Content Insights - 1 - Orchestrator","Facebook Content Insights - 2 - Wrapper","Facebook Content Insights - 3a - Date","Facebook Content Insights - 3 - Stage Object","Facebook Content Insights - 4 - Query Source"],"transformationJobs":[],"imageMetadata":{"componentIcon":"iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAACwElEQVR42u2X709SYRTH/R/aete/0QujaXNZuWayuZLfPxMFggTm9XoxQ0RFTQMCRbC2RIvwR3O0NDcqW+ZqsZXp1qYvLGiNrRf1D5ye5zGYFBe28l580dm+4z7s7p7PPec8556nouK/7bPj8ltHTjQFxZXiCSUXws/GPtidi4IpJOBYqYIQmI4H53tCvv4AqBSNq/gCwOn4Z4BqeRi8U2vw+n0Ktne/wc6nPV3uWuAH4GVyFwqZyRXnHkDWHgM24wWAvrGc5zS5mYbhyRcwhNRgiHAPwIw+yQNwh57zW4S/AwxM8ASgYeZhazsDn79+zwP4kvkBWzsZIjkV4w7A1BuHUtbUdr+8ALWaO9wB1LdOgTOQgNmlD3lOF1Y2SV1gHboiPCkeLx/AKWkA1Fdc5QHAzs22a3AV6bz6JlRJxvgFuKQbApqmoQOroxMoJIl+AAQoJbylQIocGizXSR0ItSNgoxjQmHq5BVAZ+6BG5s+tG5Dj7PUZhY9Eo8Xi0LECVElDUC0L/xUAzjsO+1mll72bogige6KsAHifm9En1R1aBdfYU7g99xb6gs/I2nN3jbRjNoA6lYcACApsvVxttAzie5KsAHiisfQ/gh5/AvSORejyrAA9skygcCcUWaOgtc/D0urHnIw9i+ThtSjEGABHgg0A1wW6J8EKcEEfyet8+LfRNAM1ikmSnmKpESBZ2+0gQm+Z/Q9vv/0RaTY7gaJoOysAfmPcWr0o3P7pdQjH3sC9+Dt48HgDfJFXJSeexuZhsuXqNaNkfQ7VgxblPQuBgU7L/ErWsVyIJhoFNQvqzjnyaQ3MrJO0iG1R0HU/BKFxuuQukbS6SSr0bY5s0eVBFBzLfx1M0gc1euMtJzf0gxH1AQxgsnZnu2G6+OkIH81QOg5KF3WDTgSwUafyqYoezbgyhmGO2Wy2o4f2QPwT2xYij2Ko+dMAAAAASUVORK5CYII=","smallIcon":"iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAABX0lEQVR42mNgoAawjZjhbB0xYx0QbyASrwPpgRtgFT5jPVDwP4l4HdwAqKkYipZtufj/1dsv/99//P4/OGcpuvwGvAYE5y79DwKfv/78f/Tcw/9+GYtIMyC6ZCXYgCNnH+DyAm4DgnKWgJ0NAr9+//3/+t3X/95pC4k3wDd94f/Tl5+ADXj9/uv/Pcfv/reLnkWaF2KweME2csZ/17jJ5BuQkN36P6uw4X9wag/pBrjFTfrvEjvlv29S//+glN7/aXlNm1AMiC1d9T+xcs3/9Lr1/1Nq1v1Pq1kPN8Axetr/uKw2FAsyC+vvoRgQWbjifxzQkCSgIZGFy4EBueh/Ve/O/+m16//bRU3/n5jTAg8HsAEFDedRknJF947/VX07/09YcPR/w+Q9GAknIr3zf3Rmx/+k3Bagdyb/t4ucvg4lM0HzA95M5J3YvyG/pPZaUGrvepTMRAooKyvjReYDAPhGyaf54G3uAAAAAElFTkSuQmCC"},"description":"Inital draft - shared job for Facebook Content Insights.","helpHtml":"

Facebook Content Insights Incremental Load

Inital draft - shared job for Facebook Content Insights.

Properties

PropertyTypeDescription
OAuthVariableThe name of the OAuth entry to be used from the Matillion OAuth Manager to connect to the source. These must be set up in advance, using Project → Manage OAuth.
Target ValueVariableThis is the Id of the page or post to get Content Insights for.
Tables and ColumnsGridContains the list of tables and columns (and in some cases an incremental_column to specify which column the load should be incremented on [accepts values of 0 or 1]) to be processed.
Connection OptionsGridA list of values and parameters. Parameters and their allowed values are database/driver specific. Referring to the data model will provide insight of what you could provide here.\n
\nThey are usually not required as sensible defaults are assumed.
Initial Start DateVariable
PeriodVariableThe period for the insight. This must always be specified.\n
\nThe allowed values are day, week, days_28, lifetime.
Instagram Business Account IdVariableInstagram Business Account Id. \n
\nTarget for Instagram Content Insights.
Load TypeVariableSequential - Iterations are done in sequence, waiting for each to complete before starting the next. \nThis is the default.\n
\nConcurrent - Iterations are run concurrently. This requires all "Variables to Iterate" to be defined as\n<a href="https://redshiftsupport.matillion.com/customer/portal/articles/2037630?b_id=8915#header1" target="_blank">Copied variables</a>, so that each iteration gets its own copy of the variable isolated from the same\nvariable being used by other concurrent executions.\n<span class="env rs sf">
Note: The maximum concurrency is limited by the number of available threads (2x the number of virtual cpus on your cloud instance).
Distribution StyleVariableEven: Distributes rows around the Redshift cluster evenly.
\nAll: Copy rows to all nodes in the Redshift cluster.
\nKey: Distribute rows around the Redshift cluster according to the value of a key column.
\nTable distribution is critical to good performance. See the <a href="http://docs.aws.amazon.com/redshift/latest/dg/tutorial-tuning-tables-distribution.html" target="_blank">Amazon Redshift</a> documentation for more information.
S3 Bucket NameVariable<span class="env sf">(AWS Only) The name of an S3 bucket for temporary storage. Ensure your access credentials have S3 access and permission to write to the bucket. See <a href="https://redshiftsupport.matillion.com/customer/portal/articles/2054760" target="_blank">this document</a> for details on setting up access. The temporary objects created in this bucket will be removed again after the load completes, they are not kept.
\n<span class="env sf">This property is available when using an Existing Amazon S3 Location for Staging.
Stage PrefixVariableA prefix value that will be added to the start of the stage table names.\n
\ne.g. If a Sarget Prefix of 'stage_' is specified and the table being processed is named 'test_data' then the target table will be names 'stage_test_data'.
Stage SchemaVariableThe schema name where the staging data will be stored.
Target PrefixVariableA prefix value that will be added to the start of the target table names.\n
\ne.g. If a Target Prefix of 'target_' is specified and the table being processed is named 'test_data' then the target table will be names 'target_test_data'.
Target SchemaVariableThe schema name where the target data will be stored.
EncryptionVariable<span class="env sf">(AWS Only) Decide on how the files are encrypted inside the S3 Bucket.<span class="env sf">This property is available when using an Existing Amazon S3 Location for Staging.
\n None: No encryption.
\n SSE KMS: Encrypt the data according to a key stored on <a href="https://aws.amazon.com/kms/" target="_blank">KMS</a>.
\n SSE S3: Encrypt the data according to a key stored on an <a href="http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html" target="_blank">S3 bucket</a>
KMS Key IdVariableThe ID of the KMS encryption key you have chosen to use in the 'Encryption' property.
Log MetricsVariableOption to switch off Metrics logging in the full product - default is TRUE which records the metrics for the batch run.
","created":1621435399938},"orchestrationJobs":{"Facebook Content Insights - 3a - Date":{"id":-2,"revision":1,"created":1621415704172,"timestamp":1621415704172,"components":{"19169":{"id":19169,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":1600,"y":0,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[23924],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Start","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"19178":{"id":19178,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":1760,"y":0,"width":32,"height":32,"inputConnectorIDs":[23924],"outputSuccessConnectorIDs":[19191],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"static_row_count","mapTo":"static_row_count","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Stage Data"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Facebook Content Insights - 4 - Query Source"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"warehouse"},"2":{"slot":2,"type":"STRING","value":"${stage_warehouse}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"database"},"2":{"slot":2,"type":"STRING","value":"${stage_database}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"schema"},"2":{"slot":2,"type":"STRING","value":"${stage_schema}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"project"},"2":{"slot":2,"type":"STRING","value":"${stage_project}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"dataset"},"2":{"slot":2,"type":"STRING","value":"${stage_dataset}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"table"},"2":{"slot":2,"type":"STRING","value":"${stage_table_name}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"s3_bucket_name"},"2":{"slot":2,"type":"STRING","value":"${s3_bucket_name}"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"dist_style"},"2":{"slot":2,"type":"STRING","value":"${dist_style}"}}},"9":{"slot":9,"values":{"1":{"slot":1,"type":"STRING","value":"staging_type"},"2":{"slot":2,"type":"STRING","value":"${staging_type}"}}},"10":{"slot":10,"values":{"1":{"slot":1,"type":"STRING","value":"blob_container"},"2":{"slot":2,"type":"STRING","value":"${blob_container}"}}},"11":{"slot":11,"values":{"1":{"slot":1,"type":"STRING","value":"storage_account"},"2":{"slot":2,"type":"STRING","value":"${storage_account}"}}},"12":{"slot":12,"values":{"1":{"slot":1,"type":"STRING","value":"cloud_storage_area"},"2":{"slot":2,"type":"STRING","value":"${cloud_storage_area}"}}},"13":{"slot":13,"values":{"1":{"slot":1,"type":"STRING","value":"encryption_method"},"2":{"slot":2,"type":"STRING","value":"${encryption_method}"}}},"14":{"slot":14,"values":{"1":{"slot":1,"type":"STRING","value":"kms_key_id"},"2":{"slot":2,"type":"STRING","value":"${kms_key_id}"}}},"15":{"slot":15,"values":{"1":{"slot":1,"type":"STRING","value":"auto_debug"},"2":{"slot":2,"type":"STRING","value":"${auto_debug}"}}},"16":{"slot":16,"values":{"1":{"slot":1,"type":"STRING","value":"auto_debug_level"},"2":{"slot":2,"type":"STRING","value":"${auto_debug_level}"}}},"17":{"slot":17,"values":{"1":{"slot":1,"type":"STRING","value":"target_table_name"},"2":{"slot":2,"type":"STRING","value":"${target_table_name}"}}},"18":{"slot":18,"values":{"1":{"slot":1,"type":"STRING","value":"table_name"},"2":{"slot":2,"type":"STRING","value":"${table_name}"}}},"19":{"slot":19,"values":{"1":{"slot":1,"type":"STRING","value":"source_column_list"},"2":{"slot":2,"type":"STRING","value":"${source_column_list}"}}},"20":{"slot":20,"values":{"1":{"slot":1,"type":"STRING","value":"oauth"},"2":{"slot":2,"type":"STRING","value":"${oauth}"}}},"21":{"slot":21,"values":{"1":{"slot":1,"type":"STRING","value":"key_value"},"2":{"slot":2,"type":"STRING","value":"${key_value}"}}},"22":{"slot":22,"values":{"1":{"slot":1,"type":"STRING","value":"period"},"2":{"slot":2,"type":"STRING","value":"${period}"}}},"23":{"slot":23,"values":{"1":{"slot":1,"type":"STRING","value":"start_date"},"2":{"slot":2,"type":"STRING","value":"${start_date}"}}},"24":{"slot":24,"values":{"1":{"slot":1,"type":"STRING","value":"end_date"},"2":{"slot":2,"type":"STRING","value":"${end_date}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"advanced_connection_options"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"parameter"},"2":{"slot":2,"type":"GRID","value":"value"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"advanced_connection_options"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Facebook Content Insights Incremental Load"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Incremental"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"2"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"19179":{"id":19179,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-313802845,"x":1920,"y":0,"width":32,"height":32,"inputConnectorIDs":[19191],"outputSuccessConnectorIDs":[22143],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"logged_rows_inserted","mapTo":"logged_rows_inserted_local","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Load Target"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"platform","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_platform}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"stage_warehouse","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_warehouse}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"stage_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_database}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"stage_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"stage_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_project}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"stage_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"stage_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"target_warehouse","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_warehouse}"}}}},"visible":true,"status":{},"columnNames":[]},"10":{"slot":10,"name":"target_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"11":{"slot":11,"name":"target_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"12":{"slot":12,"name":"target_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_project}"}}}},"visible":true,"status":{},"columnNames":[]},"13":{"slot":13,"name":"target_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"14":{"slot":14,"name":"target_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"15":{"slot":15,"name":"load_strategy","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${load_strategy}"}}}},"visible":true,"status":{},"columnNames":[]},"16":{"slot":16,"name":"dist_style","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${dist_style}"}}}},"visible":true,"status":{},"columnNames":[]},"17":{"slot":17,"name":"columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"stage_table_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"column_name"}}}},"visible":true,"status":{},"columnNames":[]},"18":{"slot":18,"name":"key_columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"key_columns"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"column_name"}}}},"visible":true,"status":{},"columnNames":[]},"19":{"slot":19,"name":"incremental_column","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${incremental_column}"}}}},"visible":true,"status":{},"columnNames":[]},"20":{"slot":20,"name":"join_statement","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"21":{"slot":21,"name":"log_metrics","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"FALSE"}}}},"visible":true,"status":{},"columnNames":[]},"22":{"slot":22,"name":"static_row_count","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${static_row_count}"}}}},"visible":true,"status":{},"columnNames":[]},"23":{"slot":23,"name":"logged_rows_inserted","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"DECIMAL","value":"0"}}}},"visible":true,"status":{},"columnNames":[]},"24":{"slot":24,"name":"logged_rows_update","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"DECIMAL","value":"0"}}}},"visible":true,"status":{},"columnNames":[]},"25":{"slot":25,"name":"logged_rows_deleted","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"DECIMAL","value":"0"}}}},"visible":true,"status":{},"columnNames":[]},"26":{"slot":26,"name":"sub_RangeStartDate","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"19180":{"id":19180,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":2240,"y":0,"width":32,"height":32,"inputConnectorIDs":[22154],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Success"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"22142":{"id":22142,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":2080,"y":0,"width":32,"height":32,"inputConnectorIDs":[22143],"outputSuccessConnectorIDs":[22154],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Update Logged Row Counts"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"# BigQuery does not get row counts correctly (always pulls full table row count)\n# must supply a static row count (from the staging table) \n# which will replace the wrongly calculated variable\nif target_platform.lower()=='bigquery' and log_metrics=='TRUE':\n context.updateVariable('logged_rows_inserted_local',static_row_count)\n\n\n# need to bubble up rows inserted from this iteration and combine with\n# total rows inserted from previous iterations\nlogged_rows_inserted = int(str(logged_rows_inserted)) + int(str(logged_rows_inserted_local))\ncontext.updateVariable('logged_rows_inserted', logged_rows_inserted)\nlogged_rows_updated = int(str(logged_rows_updated)) + int(str(logged_rows_updated_local))\ncontext.updateVariable('logged_rows_updated', logged_rows_updated)\n\nmsg=\\\n\"\"\"## Metrics for {load_strategy} load into {target_table_name} ##\nInserted: {logged_rows_inserted}\nUpdated: {logged_rows_updated}\nDeleted: {logged_rows_deleted}\n\"\"\".format(\n load_strategy = load_strategy,\n target_table_name = target_table_name,\n logged_rows_inserted = logged_rows_inserted,\n logged_rows_updated = logged_rows_updated,\n logged_rows_deleted = logged_rows_deleted\n )\n\nprint(msg)"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"19191":{"id":19191,"sourceID":19178,"targetID":19179},"22143":{"id":22143,"sourceID":19179,"targetID":22142},"22154":{"id":22154,"sourceID":22142,"targetID":19180}},"failureConnectors":{},"unconditionalConnectors":{"23924":{"id":23924,"sourceID":19169,"targetID":19178}},"trueConnectors":{},"falseConnectors":{},"iterationConnectors":{},"noteConnectors":{},"canUndo":false,"undoCommand":"","undoCreated":0,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"19168":{"id":19168,"x":1685,"y":-231,"width":467,"height":303,"text":"** Load Data **\n\n- (__Stage Data__) Executes __stage_sql__ query and stores the data in the staging table \n\n- (__Load Target__) Executes logic to move the Stage Table Data into the Target Table using the desired __Load Strategy__\n\n- (__Updated Logged Row Counts__) Update the logged row count variable and export it back out","colour":"e6e63c"}},"variables":{"oauth":{"definition":{"name":"oauth","type":"TEXT","scope":"TASKBATCH","description":"The Matillion OAuth name to use when connecting to Sales Force.com","visibility":"PUBLIC"},"value":""},"period":{"definition":{"name":"period","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"end_date":{"definition":{"name":"end_date","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"key_value":{"definition":{"name":"key_value","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"auto_debug":{"definition":{"name":"auto_debug","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"dist_style":{"definition":{"name":"dist_style","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"kms_key_id":{"definition":{"name":"kms_key_id","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"start_date":{"definition":{"name":"start_date","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"table_name":{"definition":{"name":"table_name","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"log_metrics":{"definition":{"name":"log_metrics","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_schema":{"definition":{"name":"stage_schema","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"staging_type":{"definition":{"name":"staging_type","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"load_strategy":{"definition":{"name":"load_strategy","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_dataset":{"definition":{"name":"stage_dataset","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_project":{"definition":{"name":"stage_project","type":"TEXT","scope":"BRANCH","description":" ","visibility":"PUBLIC"},"value":""},"target_schema":{"definition":{"name":"target_schema","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"blob_container":{"definition":{"name":"blob_container","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"s3_bucket_name":{"definition":{"name":"s3_bucket_name","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_database":{"definition":{"name":"stage_database","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_dataset":{"definition":{"name":"target_dataset","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_project":{"definition":{"name":"target_project","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_warehouse":{"definition":{"name":"stage_warehouse","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"storage_account":{"definition":{"name":"storage_account","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_database":{"definition":{"name":"target_database","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_platform":{"definition":{"name":"target_platform","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"auto_debug_level":{"definition":{"name":"auto_debug_level","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"key_column_count":{"definition":{"name":"key_column_count","type":"DECIMAL","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_table_name":{"definition":{"name":"stage_table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"static_row_count":{"definition":{"name":"static_row_count","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"target_warehouse":{"definition":{"name":"target_warehouse","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"encryption_method":{"definition":{"name":"encryption_method","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_table_name":{"definition":{"name":"target_table_name","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"cloud_storage_area":{"definition":{"name":"cloud_storage_area","type":"TEXT","scope":"BRANCH","description":"The cloud storage area name that should be used when loading bq tables.","visibility":"PUBLIC"},"value":""},"incremental_column":{"definition":{"name":"incremental_column","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"source_column_list":{"definition":{"name":"source_column_list","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"logged_rows_deleted":{"definition":{"name":"logged_rows_deleted","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"0"},"logged_rows_updated":{"definition":{"name":"logged_rows_updated","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"},"timestamp_precision":{"definition":{"name":"timestamp_precision","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"microseconds"},"logged_rows_inserted":{"definition":{"name":"logged_rows_inserted","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"},"incremental_column_count":{"definition":{"name":"incremental_column_count","type":"DECIMAL","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"0"},"logged_rows_updated_local":{"definition":{"name":"logged_rows_updated_local","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"0"},"metadata_stage_table_name":{"definition":{"name":"metadata_stage_table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"logged_rows_inserted_local":{"definition":{"name":"logged_rows_inserted_local","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"0"},"instagram_business_account_id":{"definition":{"name":"instagram_business_account_id","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""}},"grids":{"key_columns":{"definition":{"name":"key_columns","scope":"BRANCH","definitions":[{"name":"column_name","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]},"source_list":{"definition":{"name":"source_list","scope":"BRANCH","definitions":[{"name":"table_name","type":"TEXT"},{"name":"column_name","type":"TEXT"},{"name":"incremental_column","type":"DECIMAL"},{"name":"key_column","type":"DECIMAL"}],"description":"","visibility":"PUBLIC"},"values":[]},"stage_table_metadata":{"definition":{"name":"stage_table_metadata","scope":"BRANCH","definitions":[{"name":"column_name","type":"TEXT"},{"name":"type","type":"TEXT"},{"name":"size","type":"TEXT"},{"name":"precision","type":"TEXT"},{"name":"database_type","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]},"source_table_metadata":{"definition":{"name":"source_table_metadata","scope":"BRANCH","definitions":[{"name":"column_name","type":"TEXT"},{"name":"data_type_name","type":"TEXT"},{"name":"length","type":"TEXT"},{"name":"numeric_scale","type":"TEXT"},{"name":"nullable","type":"TEXT"},{"name":"ordinal","type":"TEXT"},{"name":"is_key","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[]},"advanced_connection_options":{"definition":{"name":"advanced_connection_options","scope":"BRANCH","definitions":[{"name":"parameter","type":"TEXT"},{"name":"value","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]}}},"Facebook Content Insights - 2 - Wrapper":{"id":-2,"revision":1,"created":1621415704159,"timestamp":1621415704159,"components":{"17610":{"id":17610,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":-720,"y":0,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[18741],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Start","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"17611":{"id":17611,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":-400,"y":0,"width":32,"height":32,"inputConnectorIDs":[18741],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[17618],"outputUnconditionalConnectorIDs":[17617],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"metadata_stage_table_name","mapTo":"metadata_stage_table_name","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"stage_table_name","mapTo":"stage_table_name","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Stage Object"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Facebook Content Insights - 3 - Stage Object"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"target_platform"},"2":{"slot":2,"type":"STRING","value":"${target_platform}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"stage_warehouse"},"2":{"slot":2,"type":"STRING","value":"${stage_warehouse}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"stage_database"},"2":{"slot":2,"type":"STRING","value":"${stage_database}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"stage_schema"},"2":{"slot":2,"type":"STRING","value":"${stage_schema}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"stage_project"},"2":{"slot":2,"type":"STRING","value":"${stage_project}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"stage_dataset"},"2":{"slot":2,"type":"STRING","value":"${stage_dataset}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"stage_prefix"},"2":{"slot":2,"type":"STRING","value":"${stage_prefix}"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"target_warehouse"},"2":{"slot":2,"type":"STRING","value":"${target_warehouse}"}}},"9":{"slot":9,"values":{"1":{"slot":1,"type":"STRING","value":"target_database"},"2":{"slot":2,"type":"STRING","value":"${target_database}"}}},"10":{"slot":10,"values":{"1":{"slot":1,"type":"STRING","value":"target_schema"},"2":{"slot":2,"type":"STRING","value":"${target_schema}"}}},"11":{"slot":11,"values":{"1":{"slot":1,"type":"STRING","value":"target_project"},"2":{"slot":2,"type":"STRING","value":"${target_project}"}}},"12":{"slot":12,"values":{"1":{"slot":1,"type":"STRING","value":"target_dataset"},"2":{"slot":2,"type":"STRING","value":"${target_dataset}"}}},"13":{"slot":13,"values":{"1":{"slot":1,"type":"STRING","value":"target_prefix"},"2":{"slot":2,"type":"STRING","value":"${target_prefix}"}}},"14":{"slot":14,"values":{"1":{"slot":1,"type":"STRING","value":"table_name"},"2":{"slot":2,"type":"STRING","value":"${table_name}"}}},"15":{"slot":15,"values":{"1":{"slot":1,"type":"STRING","value":"s3_bucket_name"},"2":{"slot":2,"type":"STRING","value":"${s3_bucket_name}"}}},"16":{"slot":16,"values":{"1":{"slot":1,"type":"STRING","value":"dist_style"},"2":{"slot":2,"type":"STRING","value":"${dist_style}"}}},"17":{"slot":17,"values":{"1":{"slot":1,"type":"STRING","value":"staging_type"},"2":{"slot":2,"type":"STRING","value":"${staging_type}"}}},"18":{"slot":18,"values":{"1":{"slot":1,"type":"STRING","value":"blob_container"},"2":{"slot":2,"type":"STRING","value":"${blob_container}"}}},"19":{"slot":19,"values":{"1":{"slot":1,"type":"STRING","value":"storage_account"},"2":{"slot":2,"type":"STRING","value":"${storage_account}"}}},"20":{"slot":20,"values":{"1":{"slot":1,"type":"STRING","value":"cloud_storage_area"},"2":{"slot":2,"type":"STRING","value":"${cloud_storage_area}"}}},"21":{"slot":21,"values":{"1":{"slot":1,"type":"STRING","value":"encryption_method"},"2":{"slot":2,"type":"STRING","value":"${encryption_method}"}}},"22":{"slot":22,"values":{"1":{"slot":1,"type":"STRING","value":"kms_key_id"},"2":{"slot":2,"type":"STRING","value":"${kms_key_id}"}}},"23":{"slot":23,"values":{"1":{"slot":1,"type":"STRING","value":"auto_debug"},"2":{"slot":2,"type":"STRING","value":"${auto_debug}"}}},"24":{"slot":24,"values":{"1":{"slot":1,"type":"STRING","value":"auto_debug_level"},"2":{"slot":2,"type":"STRING","value":"${auto_debug_level}"}}},"25":{"slot":25,"values":{"1":{"slot":1,"type":"STRING","value":"log_metrics"},"2":{"slot":2,"type":"STRING","value":"${log_metrics}"}}},"26":{"slot":26,"values":{"1":{"slot":1,"type":"STRING","value":"oauth"},"2":{"slot":2,"type":"STRING","value":"${oauth}"}}},"27":{"slot":27,"values":{"1":{"slot":1,"type":"STRING","value":"key_value"},"2":{"slot":2,"type":"STRING","value":"${key_value}"}}},"28":{"slot":28,"values":{"1":{"slot":1,"type":"STRING","value":"period"},"2":{"slot":2,"type":"STRING","value":"${period}"}}},"29":{"slot":29,"values":{"1":{"slot":1,"type":"STRING","value":"instagram_business_account_id"},"2":{"slot":2,"type":"STRING","value":"${instagram_business_account_id}"}}},"30":{"slot":30,"values":{"1":{"slot":1,"type":"STRING","value":"initial_start_date"},"2":{"slot":2,"type":"STRING","value":"${initial_start_date}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"advanced_connection_options"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"parameter"},"2":{"slot":2,"type":"GRID","value":"value"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"advanced_connection_options"},"2":{"slot":2,"type":"STRING","value":"grid"}}},"2":{"slot":2,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"source_list"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"table_name"},"2":{"slot":2,"type":"GRID","value":"column_name"},"3":{"slot":3,"type":"GRID","value":"incremental_column"},"4":{"slot":4,"type":"GRID","value":"key_column"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"source_list"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Facebook Content Insights Incremental Load"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Incremental"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"2"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"17612":{"id":17612,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1291941950,"x":-240,"y":0,"width":32,"height":32,"inputConnectorIDs":[17617],"outputSuccessConnectorIDs":[17615],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Drop Stage Tables"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Tables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${metadata_stage_table_name}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Cascade","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Yes"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Ignore Missing","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Yes"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"17613":{"id":17613,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":80,"y":0,"width":32,"height":32,"inputConnectorIDs":[17615],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"17614":{"id":17614,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":-399,"y":161,"width":32,"height":32,"inputConnectorIDs":[17618],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Failure"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"17615":{"id":17615,"sourceID":17612,"targetID":17613}},"failureConnectors":{"17618":{"id":17618,"sourceID":17611,"targetID":17614}},"unconditionalConnectors":{"17617":{"id":17617,"sourceID":17611,"targetID":17612},"18741":{"id":18741,"sourceID":17610,"targetID":17611}},"trueConnectors":{},"falseConnectors":{},"iterationConnectors":{},"noteConnectors":{},"canUndo":false,"undoCommand":"","undoCreated":0,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"17609":{"id":17609,"x":-555,"y":-234,"width":470,"height":309,"text":"**Execute ETL Flow for specific source table**\n\nThis job wraps our Stage 3 job (Stage Object). This is done so that we can manage some exported variables from the Stage 3. If we were directly iterating over (Stage Object) we would not be able to handle these exported variables.\n\n- (__Stage Object__) Passes in the parameters and their values into the Stage - 3 job. Regardless of the result, the __metadata_stage_table_name__ and __stage_table_name__ are exported - this is to ensure these tables are dropped if the process fails before it has time to clean up these staging tables (when a run succeeds, these staging tables will no longer exist).\n\n- (__Drop Stage Tables__) Passes in the exported table names to be deleted. This flow is executed regardless of the result of Stage Object.","colour":"e6e63c"}},"variables":{"oauth":{"definition":{"name":"oauth","type":"TEXT","scope":"TASKBATCH","description":"The Matillion OAuth name to use when connecting to Sales Force.com","visibility":"PUBLIC"},"value":""},"period":{"definition":{"name":"period","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"key_value":{"definition":{"name":"key_value","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"auto_debug":{"definition":{"name":"auto_debug","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"dist_style":{"definition":{"name":"dist_style","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"kms_key_id":{"definition":{"name":"kms_key_id","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"table_name":{"definition":{"name":"table_name","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"log_metrics":{"definition":{"name":"log_metrics","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_prefix":{"definition":{"name":"stage_prefix","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_schema":{"definition":{"name":"stage_schema","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"staging_type":{"definition":{"name":"staging_type","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_dataset":{"definition":{"name":"stage_dataset","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"stage_project":{"definition":{"name":"stage_project","type":"TEXT","scope":"BRANCH","description":" ","visibility":"PUBLIC"},"value":"[Environment Default]"},"target_prefix":{"definition":{"name":"target_prefix","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_schema":{"definition":{"name":"target_schema","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"blob_container":{"definition":{"name":"blob_container","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"s3_bucket_name":{"definition":{"name":"s3_bucket_name","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_database":{"definition":{"name":"stage_database","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"target_dataset":{"definition":{"name":"target_dataset","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_project":{"definition":{"name":"target_project","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_warehouse":{"definition":{"name":"stage_warehouse","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"storage_account":{"definition":{"name":"storage_account","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_database":{"definition":{"name":"target_database","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_platform":{"definition":{"name":"target_platform","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"auto_debug_level":{"definition":{"name":"auto_debug_level","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_table_name":{"definition":{"name":"stage_table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"target_warehouse":{"definition":{"name":"target_warehouse","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"encryption_method":{"definition":{"name":"encryption_method","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"cloud_storage_area":{"definition":{"name":"cloud_storage_area","type":"TEXT","scope":"BRANCH","description":"The cloud storage area name that should be used when loading bq tables.","visibility":"PUBLIC"},"value":""},"initial_start_date":{"definition":{"name":"initial_start_date","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"metadata_stage_table_name":{"definition":{"name":"metadata_stage_table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"instagram_business_account_id":{"definition":{"name":"instagram_business_account_id","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""}},"grids":{"source_list":{"definition":{"name":"source_list","scope":"BRANCH","definitions":[{"name":"table_name","type":"TEXT"},{"name":"column_name","type":"TEXT"},{"name":"incremental_column","type":"DECIMAL"},{"name":"key_column","type":"DECIMAL"}],"description":"","visibility":"PUBLIC"},"values":[]},"advanced_connection_options":{"definition":{"name":"advanced_connection_options","scope":"BRANCH","definitions":[{"name":"parameter","type":"TEXT"},{"name":"value","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]}}},"Facebook Content Insights - 1 - Orchestrator":{"id":-2,"revision":1,"created":1621415704165,"timestamp":1621415704165,"components":{"17557":{"id":17557,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186960,"x":-176,"y":0,"width":32,"height":32,"inputConnectorIDs":[17572],"outputSuccessConnectorIDs":[17569],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Get Unique Table Names"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"source_list = context.getGridVariable('source_list') or []\n\n# get distinct list of table names from source_list\ndistinct_table_names = list(set([table for table, col in source_list]))\n\n# convert to list of lists to store in grid variable\ndistinct_table_names = [[name] for name in sorted(distinct_table_names)]\ncontext.updateGridVariable(\"distinct_table_names\", distinct_table_names)\n\nprint(f\"source_list: {source_list}\")\nprint(f\"distinct_table_names: {distinct_table_names}\")\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"17558":{"id":17558,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186960,"x":0,"y":0,"width":32,"height":32,"inputConnectorIDs":[17569],"outputSuccessConnectorIDs":[17570],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Update [Environment Default]"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"# Script to replace any [Environment Default] entries with the schema name\n\nif target_platform == 'Redshift':\n \n if stage_schema == '[Environment Default]':\n context.updateVariable('stage_schema', environment_default_schema)\n \n if target_schema == '[Environment Default]':\n context.updateVariable('target_schema', environment_default_schema)\n\nelif target_platform == 'Snowflake':\n \n if stage_schema == '[Environment Default]':\n context.updateVariable('stage_schema', environment_default_schema)\n \n if target_schema == '[Environment Default]':\n context.updateVariable('target_schema', environment_default_schema)\n\n if stage_database == '[Environment Default]':\n context.updateVariable('stage_database', environment_database)\n\n if target_database == '[Environment Default]':\n context.updateVariable('target_database', environment_database)\n \nelif target_platform == 'BigQuery':\n \n if stage_dataset == '[Environment Default]':\n context.updateVariable('stage_dataset', environment_default_schema)\n\n if target_dataset == '[Environment Default]':\n context.updateVariable('target_dataset', environment_default_schema)\n\n if stage_project == '[Environment Default]':\n context.updateVariable('stage_project', environment_database)\n\n if target_project == '[Environment Default]':\n context.updateVariable('target_project', environment_database)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"17559":{"id":17559,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":160,"y":32,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"2 - Wrapper"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Facebook Content Insights - 2 - Wrapper"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"target_platform"},"2":{"slot":2,"type":"STRING","value":"${target_platform}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"stage_warehouse"},"2":{"slot":2,"type":"STRING","value":"${stage_warehouse}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"stage_database"},"2":{"slot":2,"type":"STRING","value":"${stage_database}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"stage_schema"},"2":{"slot":2,"type":"STRING","value":"${stage_schema}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"stage_project"},"2":{"slot":2,"type":"STRING","value":"${stage_project}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"stage_dataset"},"2":{"slot":2,"type":"STRING","value":"${stage_dataset}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"stage_prefix"},"2":{"slot":2,"type":"STRING","value":"${stage_prefix}"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"target_warehouse"},"2":{"slot":2,"type":"STRING","value":"${target_warehouse}"}}},"9":{"slot":9,"values":{"1":{"slot":1,"type":"STRING","value":"target_database"},"2":{"slot":2,"type":"STRING","value":"${target_database}"}}},"10":{"slot":10,"values":{"1":{"slot":1,"type":"STRING","value":"target_schema"},"2":{"slot":2,"type":"STRING","value":"${target_schema}"}}},"11":{"slot":11,"values":{"1":{"slot":1,"type":"STRING","value":"target_project"},"2":{"slot":2,"type":"STRING","value":"${target_project}"}}},"12":{"slot":12,"values":{"1":{"slot":1,"type":"STRING","value":"target_dataset"},"2":{"slot":2,"type":"STRING","value":"${target_dataset}"}}},"13":{"slot":13,"values":{"1":{"slot":1,"type":"STRING","value":"target_prefix"},"2":{"slot":2,"type":"STRING","value":"${target_prefix}"}}},"14":{"slot":14,"values":{"1":{"slot":1,"type":"STRING","value":"table_name"},"2":{"slot":2,"type":"STRING","value":"${table_name}"}}},"15":{"slot":15,"values":{"1":{"slot":1,"type":"STRING","value":"s3_bucket_name"},"2":{"slot":2,"type":"STRING","value":"${s3_bucket_name}"}}},"16":{"slot":16,"values":{"1":{"slot":1,"type":"STRING","value":"dist_style"},"2":{"slot":2,"type":"STRING","value":"${dist_style}"}}},"17":{"slot":17,"values":{"1":{"slot":1,"type":"STRING","value":"staging_type"},"2":{"slot":2,"type":"STRING","value":"${staging_type}"}}},"18":{"slot":18,"values":{"1":{"slot":1,"type":"STRING","value":"blob_container"},"2":{"slot":2,"type":"STRING","value":"${blob_container}"}}},"19":{"slot":19,"values":{"1":{"slot":1,"type":"STRING","value":"storage_account"},"2":{"slot":2,"type":"STRING","value":"${storage_account}"}}},"20":{"slot":20,"values":{"1":{"slot":1,"type":"STRING","value":"cloud_storage_area"},"2":{"slot":2,"type":"STRING","value":"${cloud_storage_area}"}}},"21":{"slot":21,"values":{"1":{"slot":1,"type":"STRING","value":"encryption_method"},"2":{"slot":2,"type":"STRING","value":"${encryption_method}"}}},"22":{"slot":22,"values":{"1":{"slot":1,"type":"STRING","value":"kms_key_id"},"2":{"slot":2,"type":"STRING","value":"${kms_key_id}"}}},"23":{"slot":23,"values":{"1":{"slot":1,"type":"STRING","value":"auto_debug"},"2":{"slot":2,"type":"STRING","value":"${auto_debug}"}}},"24":{"slot":24,"values":{"1":{"slot":1,"type":"STRING","value":"auto_debug_level"},"2":{"slot":2,"type":"STRING","value":"${auto_debug_level}"}}},"25":{"slot":25,"values":{"1":{"slot":1,"type":"STRING","value":"log_metrics"},"2":{"slot":2,"type":"STRING","value":"${log_metrics}"}}},"26":{"slot":26,"values":{"1":{"slot":1,"type":"STRING","value":"oauth"},"2":{"slot":2,"type":"STRING","value":"${oauth}"}}},"27":{"slot":27,"values":{"1":{"slot":1,"type":"STRING","value":"key_value"},"2":{"slot":2,"type":"STRING","value":"${key_value}"}}},"28":{"slot":28,"values":{"1":{"slot":1,"type":"STRING","value":"period"},"2":{"slot":2,"type":"STRING","value":"${period}"}}},"29":{"slot":29,"values":{"1":{"slot":1,"type":"STRING","value":"instagram_business_account_id"},"2":{"slot":2,"type":"STRING","value":"${instagram_business_account_id}"}}},"30":{"slot":30,"values":{"1":{"slot":1,"type":"STRING","value":"initial_start_date"},"2":{"slot":2,"type":"STRING","value":"${initial_start_date}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"advanced_connection_options"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"parameter"},"2":{"slot":2,"type":"GRID","value":"value"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"advanced_connection_options"},"2":{"slot":2,"type":"STRING","value":"grid"}}},"2":{"slot":2,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"internal_source_list"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"table"},"2":{"slot":2,"type":"GRID","value":"column"},"3":{"slot":3,"type":"GRID","value":"incremental_column"},"4":{"slot":4,"type":"GRID","value":"key_column"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"source_list"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Facebook Content Insights Incremental Load"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Incremental"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"2"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[17577]},"17560":{"id":17560,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"ITERATE","implementationID":-424773870,"x":160,"y":0,"width":32,"height":16,"inputConnectorIDs":[17570],"outputSuccessConnectorIDs":[17571,17573],"outputFailureConnectorIDs":[17576],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"Duration","mapTo":"logged_duration","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Iterate over distinct table names"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Grid Variable","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"distinct_table_names"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Grid Variable Mapping","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"table_name"},"2":{"slot":2,"type":"STRING","value":"table_name"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Break on Failure","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"No"}}}},"visible":false,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Concurrency","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${load_concurrent}"}}}},"visible":true,"status":{},"columnNames":[]},"999":{"slot":999,"name":"Record Values In Task History","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Yes"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[17577],"inputIterationConnectorIDs":[]},"17561":{"id":17561,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":-960,"y":0,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[23681],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"17562":{"id":17562,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":320,"y":0,"width":32,"height":32,"inputConnectorIDs":[17571],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Success"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"17563":{"id":17563,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":320,"y":80,"width":32,"height":32,"inputConnectorIDs":[17576],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Failure"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"17564":{"id":17564,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-2112089365,"x":480,"y":-80,"width":32,"height":32,"inputConnectorIDs":[17574],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log duration"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Metrics","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"duration"},"2":{"slot":2,"type":"STRING","value":"Numeric"},"3":{"slot":3,"type":"STRING","value":"${logged_duration}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":" "}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"17565":{"id":17565,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":320,"y":-80,"width":32,"height":32,"inputConnectorIDs":[17573],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[17574],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Metrics?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"log_metrics"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"TRUE"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"17566":{"id":17566,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186960,"x":-336,"y":0,"width":32,"height":32,"inputConnectorIDs":[23645],"outputSuccessConnectorIDs":[17572],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Copy Source List"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"fixed_grid = [\n [ x[0], x[1], '0', '0' ] for x in context.getGridVariable('source_list') or []\n]\n\ncontext.updateGridVariable('internal_source_list', fixed_grid)\nprint(\"Updated Grid Variable `source_list`:{}\".format(fixed_grid))\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"17567":{"id":17567,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":-800,"y":0,"width":32,"height":32,"inputConnectorIDs":[23681],"outputSuccessConnectorIDs":[23649],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Magic Variables"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"# this may or may not contain variables which are not valid connection options\n# and are designed to be used elsewhere within the shared job, they need to be removed from the grid \n# to prevent downstream errors \nopts = context.getGridVariable('advanced_connection_options') or [] \n\n# create a new list to store the real connection options \nfiltered = [] \n\n# magic vars is a map: lower case param name => job variable name \nmagic_vars = {\n\t'autodebug': 'auto_debug', # passed into XX Query Components for CData Debugging \n 'autodebuglevel': 'auto_debug_level', # passed into XX Query Components (default = 3)\n}\n\nif not initial_start_date:\n from datetime import datetime, timedelta\n initial_start_date = datetime.now().strftime('%Y-%m-%d') + timedelta(days = 365)\n context.updateVariable('initial_start_date', initial_start_date)\n \nprint(f\"Updated Variable `initial_start_date`: {initial_start_date}\")\n\nif opts: \n for r in opts:\n param = r[0]\n val = r[1]\n \n p = param.lower()\n # check if this is a magic variable \n # update the underlying job variable if it is \n if p in magic_vars:\n context.updateVariable(magic_vars[p], val)\n # otherwise, its a normal connection option \n else:\n filtered.append( [param, val] )\n \n context.updateGridVariable('advanced_connection_options', filtered)\n \n \n "}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"23609":{"id":23609,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":1395699082,"x":-640,"y":0,"width":32,"height":32,"inputConnectorIDs":[23649],"outputSuccessConnectorIDs":[23619],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Remove Pseudo Columns from Advanced Connection Options (if exists)"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Target Grid Variable","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"advanced_connection_options"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Key Columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"parameter"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"value"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Operation","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Not In"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Fixed/Grid","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Fixed"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Values Grid","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"PseudoColumns"},"2":{"slot":2,"type":"STRING","value":"*=*"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Source Grid Variable","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"LoadColumns"}}}},"visible":false,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Key Column Mapping","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Name"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"23610":{"id":23610,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-369209785,"x":-480,"y":0,"width":32,"height":32,"inputConnectorIDs":[23619],"outputSuccessConnectorIDs":[23645],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Add PseudoColumns to Advanced Connection Options"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Target Grid Variable","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"advanced_connection_options"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Fixed/Grid","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Fixed"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Values Grid","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"PseudoColumns"},"2":{"slot":2,"type":"STRING","value":"*=*"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Source Grid Variable","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"source_list"}}}},"visible":false,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Column Mapping","elements":{},"visible":false,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Append/Prepend","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Append"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"17569":{"id":17569,"sourceID":17557,"targetID":17558},"17570":{"id":17570,"sourceID":17558,"targetID":17560},"17571":{"id":17571,"sourceID":17560,"targetID":17562},"17572":{"id":17572,"sourceID":17566,"targetID":17557},"17573":{"id":17573,"sourceID":17560,"targetID":17565},"23619":{"id":23619,"sourceID":23609,"targetID":23610},"23645":{"id":23645,"sourceID":23610,"targetID":17566},"23649":{"id":23649,"sourceID":17567,"targetID":23609}},"failureConnectors":{"17576":{"id":17576,"sourceID":17560,"targetID":17563}},"unconditionalConnectors":{"23681":{"id":23681,"sourceID":17561,"targetID":17567}},"trueConnectors":{"17574":{"id":17574,"sourceID":17565,"targetID":17564}},"falseConnectors":{},"iterationConnectors":{"17577":{"id":17577,"sourceID":17560,"targetID":17559}},"noteConnectors":{},"canUndo":false,"undoCommand":"","undoCreated":0,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"17555":{"id":17555,"x":-1671,"y":-394,"width":624,"height":463,"text":"**Facebook Content Insights Incremental Load** \n\nRoot (Stage 1) Orchestration Job of the Matillion Incremental Load pattern. \nThis job is the entry point of the shared job and is responsible for validating and reformatting some of the supplied data.\nThe pattern aims to synchronize INSERTS and UPDATES in a particular source, to a table within the relevant Cloud Data Warehouse.\n\n**Pattern Overview**\n\nThere are 4 distinct stages, each stage is represented by an orchestration job using the naming convention:\n__** - - **__\n\n+ **Stage 1** - performs input validation and prepares the variables for **Stage 3**\n\n+ **Stage 2** - wrapper for **Stage 3** to ensure that in the case of any errors, relevant staging tables are nicely cleaned up.\n\n+ **Stage 3** - Orchestration job containing the ETL logic for a single source table. This involves creating a list of date ranges to iterate over, grabbing the data in batches, rather than everything being staged in 1 request - this is due to limitations on Date Filters by Facebook.\n\n+ **Stage 3a** - Orchestration job which handles the staging and loading of the data for the specified date range.\n\n+ **Stage 4** - wrapper orchestration job to Query the underlying source. This is abstracted away from the rest of the logic to make the pattern more reusable.\n\nMore detailed information and specifics relating to each connector can be found within each job.\n\n\n======================================================================================================================================================================================================","colour":"00ce4f"},"17556":{"id":17556,"x":-875,"y":-315,"width":1110,"height":389,"text":"** Prepare User Input **\n\n- (__Magic Variables__) Script that extracts some special variables from connection options. These are used internally to allow us to use 'hidden' variables that will not affect the slots of the shared job. Currently, this is used to switch on CData logging.\n\n- (__Remove/Add Pseudo Columns__) Facebook Content Insights data model uses Pseudo Columns as key and incremental, so we need to ensure they are always in the connection options (and not duplicated)\n\n- (__Copy Source List__) Script that reformats the __source_list__ grid variable so make sure it has 4 columns.\n\n- (__Get Unique Table Names__) Script to extract a list of table names from the source. This list is then iterated over, and the same ETL logic is applied to each source. This information is stored in the __distinct_table_names__ grid variable.\n\n- (__Update [Environment Default]__) Script to replace any [Environment Default] values with their actual value.\n\n- (__2 - Wrapper__) Iterate over each of the __distinct_table_names__ and call the lower level ETL process for each table in the source.","colour":"e6e63c"}},"variables":{"oauth":{"definition":{"name":"oauth","type":"TEXT","scope":"TASKBATCH","description":"The Matillion OAuth name to use when connecting to Sales Force.com","visibility":"PUBLIC"},"value":""},"period":{"definition":{"name":"period","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"key_value":{"definition":{"name":"key_value","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"auto_debug":{"definition":{"name":"auto_debug","type":"TEXT","scope":"BRANCH","description":"","visibility":"PRIVATE"},"value":"Off"},"dist_style":{"definition":{"name":"dist_style","type":"TEXT","scope":"BRANCH","description":"Even: Distributes rows around the Redshift cluster evenly.
\nAll: Copy rows to all nodes in the Redshift cluster.
\nTable distribution is critical to good performance. See the Amazon Redshift documentation for more information.","visibility":"PUBLIC"},"value":""},"kms_key_id":{"definition":{"name":"kms_key_id","type":"TEXT","scope":"BRANCH","description":"The ID of the KMS encryption key you have chosen to use in the 'Encryption' property.","visibility":"PUBLIC"},"value":""},"table_name":{"definition":{"name":"table_name","type":"TEXT","scope":"BRANCH","description":"","visibility":"PRIVATE"},"value":""},"log_metrics":{"definition":{"name":"log_metrics","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"TRUE"},"stage_prefix":{"definition":{"name":"stage_prefix","type":"TEXT","scope":"BRANCH","description":"A prefix value that will be added to the start of the stage table names.\n
\ne.g. If a Stage Prefix of 'stage_' is specified and the table being processed is named 'test_data' then the target table will be named 'stage_test_data'.","visibility":"PUBLIC"},"value":""},"stage_schema":{"definition":{"name":"stage_schema","type":"TEXT","scope":"BRANCH","description":"The schema name where the staging data will be stored.","visibility":"PUBLIC"},"value":""},"staging_type":{"definition":{"name":"staging_type","type":"TEXT","scope":"BRANCH","description":"(AWS Only) Snowflake Managed: Allow Matillion ETL to create and use a temporary internal stage on Snowflake for staging the data. This stage, along with the staged data, will cease to exist after loading is complete.
\n Existing Amazon S3 Location: Selecting this will avail the user of properties to specify a custom staging area on S3.","visibility":"PRIVATE"},"value":"Snowflake Managed"},"stage_dataset":{"definition":{"name":"stage_dataset","type":"TEXT","scope":"BRANCH","description":"The dataset name where the staging data will be stored.","visibility":"PRIVATE"},"value":""},"stage_project":{"definition":{"name":"stage_project","type":"TEXT","scope":"BRANCH","description":"The project name where the staging data will be stored.","visibility":"PRIVATE"},"value":""},"target_prefix":{"definition":{"name":"target_prefix","type":"TEXT","scope":"BRANCH","description":"A prefix value that will be added to the start of the target table names.\n
\ne.g. If a Target Prefix of 'target_' is specified and the table being processed is named 'test_data' then the target table will be named 'target_test_data'.","visibility":"PUBLIC"},"value":""},"target_schema":{"definition":{"name":"target_schema","type":"TEXT","scope":"BRANCH","description":"The schema name where the target data will be stored.","visibility":"PUBLIC"},"value":""},"blob_container":{"definition":{"name":"blob_container","type":"TEXT","scope":"BRANCH","description":"(Azure Only) Azure Blob Container to allow Matillion ETL to use the blob storage container object on Snowflake for staging data. ","visibility":"PRIVATE"},"value":""},"s3_bucket_name":{"definition":{"name":"s3_bucket_name","type":"TEXT","scope":"BRANCH","description":"(AWS Only) The name of an S3 bucket for temporary storage. Ensure your access credentials have S3 access and permission to write to the bucket. See this document for details on setting up access. The temporary objects created in this bucket will be removed again after the load completes, they are not kept.
\nThis property is available when using an Existing Amazon S3 Location for Staging.","visibility":"PUBLIC"},"value":""},"stage_database":{"definition":{"name":"stage_database","type":"TEXT","scope":"BRANCH","description":"The database name where the staging data will be stored.","visibility":"PRIVATE"},"value":""},"target_dataset":{"definition":{"name":"target_dataset","type":"TEXT","scope":"BRANCH","description":"The dataset name where the target data will be stored.","visibility":"PRIVATE"},"value":""},"target_project":{"definition":{"name":"target_project","type":"TEXT","scope":"BRANCH","description":"The project name where the target data will be stored.","visibility":"PRIVATE"},"value":""},"load_concurrent":{"definition":{"name":"load_concurrent","type":"TEXT","scope":"BRANCH","description":"Sequential - Iterations are done in sequence, waiting for each to complete before starting the next. \nThis is the default.\n
\nConcurrent - Iterations are run concurrently. This requires all \"Variables to Iterate\" to be defined as\nCopied variables, so that each iteration gets its own copy of the variable isolated from the same\nvariable being used by other concurrent executions.\n
Note: The maximum concurrency is limited by the number of available threads (2x the number of virtual cpus on your cloud instance).","visibility":"PUBLIC"},"value":"Concurrent"},"logged_duration":{"definition":{"name":"logged_duration","type":"DECIMAL","scope":"BRANCH","description":"","visibility":"PRIVATE"},"value":""},"stage_warehouse":{"definition":{"name":"stage_warehouse","type":"TEXT","scope":"BRANCH","description":"The warehouse name where the staging data will be stored.","visibility":"PRIVATE"},"value":""},"storage_account":{"definition":{"name":"storage_account","type":"TEXT","scope":"BRANCH","description":"(Azure Only) Azure Storage Account to allow Matillion ETL to specify a blob container object on Snowflake for staging data. ","visibility":"PRIVATE"},"value":""},"target_database":{"definition":{"name":"target_database","type":"TEXT","scope":"BRANCH","description":"The database name where the target data will be stored.","visibility":"PRIVATE"},"value":""},"target_platform":{"definition":{"name":"target_platform","type":"TEXT","scope":"BRANCH","description":"","visibility":"PRIVATE"},"value":"Redshift"},"auto_debug_level":{"definition":{"name":"auto_debug_level","type":"TEXT","scope":"BRANCH","description":"","visibility":"PRIVATE"},"value":"3"},"target_warehouse":{"definition":{"name":"target_warehouse","type":"TEXT","scope":"BRANCH","description":"The warehouse name where the target data will be stored.","visibility":"PRIVATE"},"value":""},"encryption_method":{"definition":{"name":"encryption_method","type":"TEXT","scope":"BRANCH","description":"(AWS Only) Decide on how the files are encrypted inside the S3 Bucket.This property is available when using an Existing Amazon S3 Location for Staging.
\n None: No encryption.
\n SSE KMS: Encrypt the data according to a key stored on KMS.
\n SSE S3: Encrypt the data according to a key stored on an S3 bucket","visibility":"PUBLIC"},"value":"None"},"cloud_storage_area":{"definition":{"name":"cloud_storage_area","type":"TEXT","scope":"BRANCH","description":"The URL and path of the target Google Storage bucket to be used for staging the queried data throughout the job.","visibility":"PRIVATE"},"value":""},"initial_start_date":{"definition":{"name":"initial_start_date","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"instagram_business_account_id":{"definition":{"name":"instagram_business_account_id","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""}},"grids":{"source_list":{"definition":{"name":"source_list","scope":"BRANCH","definitions":[{"name":"table","type":"TEXT"},{"name":"column","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]},"insight_name_grid":{"definition":{"name":"insight_name_grid","scope":"BRANCH","definitions":[{"name":"table","type":"TEXT"},{"name":"insightname_list","type":"TEXT"},{"name":"period_required","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[{"values":["insightsbyconsumptiontype","('PAGE_CONSUMPTIONS_BY_CONSUMPTION_TYPE','POST_CLICKS_BY_TYPE','POST_CLICKS_BY_TYPE_UNIQUE')","Y"]},{"values":["insightsbyfeedbacktype","('PAGE_NEGATIVE_FEEDBACK_BY_TYPE','PAGE_NEGATIVE_FEEDBACK_BY_TYPE_UNIQUE','PAGE_POSITIVE_FEEDBACK_BY_TYPE','PAGE_POSITIVE_FEEDBACK_BY_TYPE_UNIQUE','POST_NEGATIVE_FEEDBACK_BY_TYPE','POST_NEGATIVE_FEEDBACK_BY_TYPE_UNIQUE')","Y"]},{"values":["insightsbylikesourcetype","('PAGE_FANS_BY_LIKE_SOURCE','PAGE_FANS_BY_LIKE_SOURCE_UNIQUE')","N"]},{"values":["insightsbyreactiontotals","('PAGE_ACTIONS_POST_REACTIONS_TOTAL')","N"]},{"values":["insightsbystorytype"," ('PAGE_CONTENT_ACTIVITY_BY_ACTION_TYPE','PAGE_IMPRESSIONS_BY_STORY_TYPE','PAGE_IMPRESSIONS_BY_STORY_TYPE_UNIQUE','POST_IMPRESSIONS_BY_STORY_TYPE','POST_IMPRESSIONS_BY_STORY_TYPE_UNIQUE')","Y"]},{"values":["insightsbytabtype","('PAGE_TAB_VIEWS_LOGIN_TOP_UNIQUE','PAGE_TAB_VIEWS_LOGIN_TOP','PAGE_TAB_VIEWS_LOGOUT_TOP')","Y"]},{"values":["insightsbyunlikesourcetype","('PAGE_FANS_BY_UNLIKE_SOURCE','PAGE_FANS_BY_UNLIKE_SOURCE_UNIQUE')","N"]},{"values":["instagramaccountinsights","('IMPRESSIONS','REACH','FOLLOWER_COUNT','EMAIL_CONTACTS','PHONE_CALL_CLICKS','TEXT_MESSAGE_CLICKS','GET_DIRECTIONS_CLICKS','WEBSITE_CLICKS','PROFILE_VIEWS','AUDIENCE_GENDER_AGE','AUDIENCE_LOCALE','AUDIENCE_COUNTRY','AUDIENCE_CITY','ONLINE_FOLLOWERS')","Y"]}]},"distinct_table_names":{"definition":{"name":"distinct_table_names","scope":"BRANCH","definitions":[{"name":"table_name","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[]},"internal_source_list":{"definition":{"name":"internal_source_list","scope":"BRANCH","definitions":[{"name":"table","type":"TEXT"},{"name":"column","type":"TEXT"},{"name":"incremental_column","type":"DECIMAL"},{"name":"key_column","type":"DECIMAL"}],"description":"","visibility":"PRIVATE"},"values":[]},"advanced_connection_options":{"definition":{"name":"advanced_connection_options","scope":"BRANCH","definitions":[{"name":"parameter","type":"TEXT"},{"name":"value","type":"TEXT"}],"description":"A list of values and parameters. Parameters and their allowed values are database/driver specific. Referring to the data model will provide insight of what you could provide here.\n
\nThey are usually not required as sensible defaults are assumed.","visibility":"PUBLIC"},"values":[]}}},"Facebook Content Insights - 4 - Query Source":{"id":-2,"revision":2,"created":1621429462576,"timestamp":1621429462576,"components":{"14989":{"id":14989,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-560018399,"x":160,"y":0,"width":32,"height":32,"inputConnectorIDs":[17138],"outputSuccessConnectorIDs":[15108],"outputFailureConnectorIDs":[15105],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"Row Count","mapTo":"static_row_count","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"Message","mapTo":"error_message","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Query Facebook"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Basic/Advanced Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Advanced"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Authentication","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${oauth}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Data Source","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Data Selection","elements":{},"visible":false,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Data Source Filter","elements":{},"visible":false,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Combine Filters","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":false,"status":{},"columnNames":[]},"10":{"slot":10,"name":"SQL Query","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${sql}"}}}},"visible":true,"status":{},"columnNames":[]},"11":{"slot":11,"name":"Limit","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"100"}}}},"visible":false,"status":{},"columnNames":[]},"12":{"slot":12,"name":"Connection Options","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"advanced_connection_options"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"parameter"},"2":{"slot":2,"type":"GRID","value":"value"}}}},"visible":true,"status":{},"columnNames":[]},"13":{"slot":13,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${table}"}}}},"visible":true,"status":{},"columnNames":[]},"14":{"slot":14,"name":"Distribution Style","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${dist_style}"}}}},"visible":true,"status":{},"columnNames":[]},"15":{"slot":15,"name":"Distribution Key","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"16":{"slot":16,"name":"Sort Key","elements":{},"visible":true,"status":{},"columnNames":[]},"17":{"slot":17,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${schema}"}}}},"visible":true,"status":{},"columnNames":[]},"18":{"slot":18,"name":"Sort Key Options","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"19":{"slot":19,"name":"S3 Staging Area","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${s3_bucket_name}"}}}},"visible":true,"status":{},"columnNames":[]},"20":{"slot":20,"name":"Data Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"21":{"slot":21,"name":"Primary Key","elements":{},"visible":true,"status":{},"columnNames":[]},"95":{"slot":95,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"96":{"slot":96,"name":"Location","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"s3:///"}}}},"visible":false,"status":{},"columnNames":[]},"97":{"slot":97,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"98":{"slot":98,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"99":{"slot":99,"name":"Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Standard"}}}},"visible":true,"status":{},"columnNames":[]},"1001":{"slot":1001,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"1013":{"slot":1013,"name":"Auto Debug","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${auto_debug}"}}}},"visible":true,"status":{},"columnNames":[]},"1014":{"slot":1014,"name":"Debug Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${auto_debug_level}"}}}},"visible":false,"status":{},"columnNames":[]},"1034":{"slot":1034,"name":"Load Options","elements":{},"visible":true,"status":{},"columnNames":[]},"40000":{"slot":40000,"name":"Encryption","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${encryption_method}"}}}},"visible":true,"status":{},"columnNames":[]},"40001":{"slot":40001,"name":"KMS Key ID","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${kms_key_id}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"17134":{"id":17134,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":-2,"y":-6,"width":32,"height":32,"inputConnectorIDs":[17135],"outputSuccessConnectorIDs":[17138],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Generate Filters"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"predicates = [\n\tf\"Target = '{key_value}'\"\n] \n\nfor rows in context.getGridVariable(\"insight_names\"):\n if table_name.lower() != rows[0]:\n continue \n \n predicates.append(f\"InsightName IN {rows[1]}\")\n \n if rows[2] == 'Y':\n predicates.append(f\"Period='{period}'\")\n \n \nif start_date:\n predicates.append(f\"FromDateTime='{start_date}'\")\n \n if not end_date:\n end_date = datetime.now().strftime('%Y-%m-%d')\n \n predicates.append(f\"ToDateTime='{end_date}'\")\n\n \nwhere_clause = \"\\nAND \".join(predicates)\nquery = \\\nf\"\"\"\nSELECT \n\t{source_column_list}\nFROM\n\t\"{table_name}\"\nWHERE\n\t{where_clause}\n\"\"\"\n\n \nQ = \"\" \n\nif not sql:\n Q = query \nelse:\n Q = sql \n \nif limit > 0:\n Q += f\"\\n LIMIT {limit}\"\n\n \ncontext.updateVariable('sql', Q)\n\nprint(f\"Updated `SQL` variable: {query}\")"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"17528":{"id":17528,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":-160,"y":0,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[17135],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Start","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"17529":{"id":17529,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":166,"y":160,"width":32,"height":32,"inputConnectorIDs":[15105],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[17552],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Print Error"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"msg = \\\n\"\"\"There was an issue whilst retrieving data from the source. \nThe attempted query was: \\n{q}\nError: {e}\n\"\"\".format(q = sql, e = error_message) \nprint(msg)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"17530":{"id":17530,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1334993426,"x":160,"y":320,"width":32,"height":32,"inputConnectorIDs":[17552],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[17551],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Error"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"error"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Messages","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Unable to retrieve data from source"},"2":{"slot":2,"type":"STRING","value":"${error_message}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"17531":{"id":17531,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":320,"y":320,"width":32,"height":32,"inputConnectorIDs":[17551],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Failure"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"17532":{"id":17532,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":320,"y":0,"width":32,"height":32,"inputConnectorIDs":[15108],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"18465":{"id":18465,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1407986718,"x":0,"y":160,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"Message","mapTo":"error_message","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"Row Count","mapTo":"static_row_count","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Query Facebook (Deprecated)"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Basic/Advanced Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Advanced"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Authentication","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${oauth}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Data Source","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Data Selection","elements":{},"visible":false,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Data Source Filter","elements":{},"visible":false,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Combine Filters","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":false,"status":{},"columnNames":[]},"10":{"slot":10,"name":"SQL Query","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${sql}"}}}},"visible":true,"status":{},"columnNames":[]},"11":{"slot":11,"name":"Limit","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"100"}}}},"visible":false,"status":{},"columnNames":[]},"12":{"slot":12,"name":"Connection Options","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"advanced_connection_options"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"parameter"},"2":{"slot":2,"type":"GRID","value":"value"}}}},"visible":true,"status":{},"columnNames":[]},"13":{"slot":13,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${table}"}}}},"visible":true,"status":{},"columnNames":[]},"14":{"slot":14,"name":"Distribution Style","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${dist_style}"}}}},"visible":true,"status":{},"columnNames":[]},"15":{"slot":15,"name":"Distribution Key","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"16":{"slot":16,"name":"Sort Key","elements":{},"visible":true,"status":{},"columnNames":[]},"17":{"slot":17,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${schema}"}}}},"visible":true,"status":{},"columnNames":[]},"18":{"slot":18,"name":"Sort Key Options","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"19":{"slot":19,"name":"S3 Staging Area","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${s3_bucket_name}"}}}},"visible":true,"status":{},"columnNames":[]},"20":{"slot":20,"name":"Data Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Facebook"}}}},"visible":false,"status":{},"columnNames":[]},"21":{"slot":21,"name":"Primary Key","elements":{},"visible":true,"status":{},"columnNames":[]},"95":{"slot":95,"name":"File Format","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Delimited"}}}},"visible":false,"status":{},"columnNames":[]},"96":{"slot":96,"name":"Location","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"s3:///"}}}},"visible":false,"status":{},"columnNames":[]},"97":{"slot":97,"name":"Target Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"98":{"slot":98,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]},"99":{"slot":99,"name":"Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Standard"}}}},"visible":true,"status":{},"columnNames":[]},"1001":{"slot":1001,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"1013":{"slot":1013,"name":"Auto Debug","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${auto_debug}"}}}},"visible":true,"status":{},"columnNames":[]},"1014":{"slot":1014,"name":"Debug Level","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${auto_debug_level}"}}}},"visible":false,"status":{},"columnNames":[]},"1034":{"slot":1034,"name":"Load Options","elements":{},"visible":true,"status":{},"columnNames":[]},"40000":{"slot":40000,"name":"Encryption","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${encryption_method}"}}}},"visible":true,"status":{},"columnNames":[]},"40001":{"slot":40001,"name":"KMS Key ID","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${kms_key_id}"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"DISABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{"15108":{"id":15108,"sourceID":14989,"targetID":17532},"17138":{"id":17138,"sourceID":17134,"targetID":14989}},"failureConnectors":{"15105":{"id":15105,"sourceID":14989,"targetID":17529}},"unconditionalConnectors":{"17135":{"id":17135,"sourceID":17528,"targetID":17134},"17551":{"id":17551,"sourceID":17530,"targetID":17531},"17552":{"id":17552,"sourceID":17529,"targetID":17530}},"trueConnectors":{},"falseConnectors":{},"iterationConnectors":{},"noteConnectors":{},"canUndo":true,"undoCommand":"Update Job Grid Variables","undoCreated":1621415704191,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"17527":{"id":17527,"x":-238,"y":-237,"width":633,"height":634,"text":"**Query Source**\n\n+ (__Generate Filters__) ContentInsights require specific filters that need to be applied to the WHERE clause. These include:\n- (required) InsightName - provided by User\n- (required) Target - provided by User\n- Period - provided by User\n- Start and End DateTime - generated by Job \n\n+ (__Query Facebook__) Executed the generated **sql** variable.\n\n+ (__Error Handling__) If the Query Component fails, the resulting message is printed and then written to the metrics store.","colour":"e6e63c"}},"variables":{"sql":{"definition":{"name":"sql","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":null},"limit":{"definition":{"name":"limit","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"},"oauth":{"definition":{"name":"oauth","type":"TEXT","scope":"TASKBATCH","description":"The Matillion OAuth name to use when connecting to Sales Force.com","visibility":"PUBLIC"},"value":""},"table":{"definition":{"name":"table","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"period":{"definition":{"name":"period","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"schema":{"definition":{"name":"schema","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"dataset":{"definition":{"name":"dataset","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"project":{"definition":{"name":"project","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"database":{"definition":{"name":"database","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"end_date":{"definition":{"name":"end_date","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"key_value":{"definition":{"name":"key_value","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"warehouse":{"definition":{"name":"warehouse","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"[Environment Default]"},"auto_debug":{"definition":{"name":"auto_debug","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"Off"},"dist_style":{"definition":{"name":"dist_style","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"Even"},"kms_key_id":{"definition":{"name":"kms_key_id","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"start_date":{"definition":{"name":"start_date","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"table_name":{"definition":{"name":"table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"staging_type":{"definition":{"name":"staging_type","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"Snowflake Managed"},"error_message":{"definition":{"name":"error_message","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"blob_container":{"definition":{"name":"blob_container","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"s3_bucket_name":{"definition":{"name":"s3_bucket_name","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"storage_account":{"definition":{"name":"storage_account","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"auto_debug_level":{"definition":{"name":"auto_debug_level","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"3"},"static_row_count":{"definition":{"name":"static_row_count","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"encryption_method":{"definition":{"name":"encryption_method","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":"None"},"target_table_name":{"definition":{"name":"target_table_name","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"cloud_storage_area":{"definition":{"name":"cloud_storage_area","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"source_column_list":{"definition":{"name":"source_column_list","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""}},"grids":{"insight_names":{"definition":{"name":"insight_names","scope":"BRANCH","definitions":[{"name":"table","type":"TEXT"},{"name":"names","type":"TEXT"},{"name":"req","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[{"values":["insightsbyconsumptiontype","('PAGE_CONSUMPTIONS_BY_CONSUMPTION_TYPE','POST_CLICKS_BY_TYPE','POST_CLICKS_BY_TYPE_UNIQUE','PAGE_CONSUMPTIONS_BY_CONSUMPTION_TYPE_UNIQUE')","Y"]},{"values":["insightsbyfeedbacktype","('PAGE_NEGATIVE_FEEDBACK_BY_TYPE','PAGE_NEGATIVE_FEEDBACK_BY_TYPE_UNIQUE','PAGE_POSITIVE_FEEDBACK_BY_TYPE','PAGE_POSITIVE_FEEDBACK_BY_TYPE_UNIQUE','POST_NEGATIVE_FEEDBACK_BY_TYPE','POST_NEGATIVE_FEEDBACK_BY_TYPE_UNIQUE')","Y"]},{"values":["insightsbylikesourcetype","('PAGE_FANS_BY_LIKE_SOURCE','PAGE_FANS_BY_LIKE_SOURCE_UNIQUE')","N"]},{"values":["insightsbyreactiontotals","('PAGE_ACTIONS_POST_REACTIONS_TOTAL','POST_REACTIONS_BY_TYPE_TOTAL')","N"]},{"values":["insightsbystorytype"," ('PAGE_CONTENT_ACTIVITY_BY_ACTION_TYPE','PAGE_IMPRESSIONS_BY_STORY_TYPE','PAGE_IMPRESSIONS_BY_STORY_TYPE_UNIQUE','POST_IMPRESSIONS_BY_STORY_TYPE','POST_IMPRESSIONS_BY_STORY_TYPE_UNIQUE')","Y"]},{"values":["insightsbytabtype","('PAGE_TAB_VIEWS_LOGIN_TOP_UNIQUE','PAGE_TAB_VIEWS_LOGIN_TOP','PAGE_TAB_VIEWS_LOGOUT_TOP')","Y"]},{"values":["insightsbyunlikesourcetype","('PAGE_FANS_BY_UNLIKE_SOURCE','PAGE_FANS_BY_UNLIKE_SOURCE_UNIQUE')","N"]},{"values":["instagramaccountinsights","('IMPRESSIONS','REACH','FOLLOWER_COUNT','EMAIL_CONTACTS','PHONE_CALL_CLICKS','TEXT_MESSAGE_CLICKS','GET_DIRECTIONS_CLICKS','WEBSITE_CLICKS','PROFILE_VIEWS','AUDIENCE_GENDER_AGE','AUDIENCE_LOCALE','AUDIENCE_COUNTRY','AUDIENCE_CITY','ONLINE_FOLLOWERS')","Y"]}]},"advanced_connection_options":{"definition":{"name":"advanced_connection_options","scope":"BRANCH","definitions":[{"name":"parameter","type":"TEXT"},{"name":"value","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]}}},"Facebook Content Insights - 3 - Stage Object":{"id":-2,"revision":1,"created":1621415704178,"timestamp":1621415704178,"components":{"16215":{"id":16215,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"CONDITIONAL","executionHint":"FLOW","implementationID":-1357378929,"x":1760,"y":-80,"width":32,"height":32,"inputConnectorIDs":[16231],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[16274],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Metrics?"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Mode","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Simple"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"log_metrics"},"2":{"slot":2,"type":"STRING","value":"Is"},"3":{"slot":3,"type":"STRING","value":"Equal to"},"4":{"slot":4,"type":"STRING","value":"TRUE"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Combine Conditions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"And"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Condition","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"16233":{"id":16233,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":515156205,"x":1920,"y":80,"width":32,"height":32,"inputConnectorIDs":[13531],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Failure"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"16267":{"id":16267,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-2112089365,"x":1920,"y":-80,"width":32,"height":32,"inputConnectorIDs":[16274],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Log Metrics"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Metrics","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"inserted"},"2":{"slot":2,"type":"STRING","value":"Numeric"},"3":{"slot":3,"type":"STRING","value":"${logged_rows_inserted}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"updated"},"2":{"slot":2,"type":"STRING","value":"Numeric"},"3":{"slot":3,"type":"STRING","value":"${logged_rows_updated}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"deleted"},"2":{"slot":2,"type":"STRING","value":"Numeric"},"3":{"slot":3,"type":"STRING","value":"${logged_rows_deleted}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Namespace","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"17583":{"id":17583,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":-480,"y":0,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[18140],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Start","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"17584":{"id":17584,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":0,"y":0,"width":32,"height":32,"inputConnectorIDs":[18137],"outputSuccessConnectorIDs":[17597],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Get Source Metadata"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Facebook Content Insights - 4 - Query Source"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"sql"},"2":{"slot":2,"type":"STRING","value":"${source_metadata_sql}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"warehouse"},"2":{"slot":2,"type":"STRING","value":"${stage_warehouse}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"database"},"2":{"slot":2,"type":"STRING","value":"${stage_database}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"schema"},"2":{"slot":2,"type":"STRING","value":"${stage_schema}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"project"},"2":{"slot":2,"type":"STRING","value":"${stage_project}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"dataset"},"2":{"slot":2,"type":"STRING","value":"${stage_dataset}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"table"},"2":{"slot":2,"type":"STRING","value":"${metadata_stage_table_name}"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"s3_bucket_name"},"2":{"slot":2,"type":"STRING","value":"${s3_bucket_name}"}}},"9":{"slot":9,"values":{"1":{"slot":1,"type":"STRING","value":"dist_style"},"2":{"slot":2,"type":"STRING","value":"${dist_style}"}}},"10":{"slot":10,"values":{"1":{"slot":1,"type":"STRING","value":"staging_type"},"2":{"slot":2,"type":"STRING","value":"${staging_type}"}}},"11":{"slot":11,"values":{"1":{"slot":1,"type":"STRING","value":"blob_container"},"2":{"slot":2,"type":"STRING","value":"${blob_container}"}}},"12":{"slot":12,"values":{"1":{"slot":1,"type":"STRING","value":"storage_account"},"2":{"slot":2,"type":"STRING","value":"${storage_account}"}}},"13":{"slot":13,"values":{"1":{"slot":1,"type":"STRING","value":"cloud_storage_area"},"2":{"slot":2,"type":"STRING","value":"${cloud_storage_area}"}}},"14":{"slot":14,"values":{"1":{"slot":1,"type":"STRING","value":"encryption_method"},"2":{"slot":2,"type":"STRING","value":"${encryption_method}"}}},"15":{"slot":15,"values":{"1":{"slot":1,"type":"STRING","value":"kms_key_id"},"2":{"slot":2,"type":"STRING","value":"${kms_key_id}"}}},"16":{"slot":16,"values":{"1":{"slot":1,"type":"STRING","value":"auto_debug"},"2":{"slot":2,"type":"STRING","value":"${auto_debug}"}}},"17":{"slot":17,"values":{"1":{"slot":1,"type":"STRING","value":"auto_debug_level"},"2":{"slot":2,"type":"STRING","value":"${auto_debug_level}"}}},"18":{"slot":18,"values":{"1":{"slot":1,"type":"STRING","value":"target_table_name"},"2":{"slot":2,"type":"STRING","value":"${target_table_name}"}}},"19":{"slot":19,"values":{"1":{"slot":1,"type":"STRING","value":"oauth"},"2":{"slot":2,"type":"STRING","value":"${oauth}"}}},"20":{"slot":20,"values":{"1":{"slot":1,"type":"STRING","value":"table_name"},"2":{"slot":2,"type":"STRING","value":"${table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"advanced_connection_options"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"parameter"},"2":{"slot":2,"type":"GRID","value":"value"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"advanced_connection_options"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Facebook Content Insights Incremental Load"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Incremental"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"2"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"17585":{"id":17585,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":1010676649,"x":-320,"y":0,"width":32,"height":32,"inputConnectorIDs":[18140],"outputSuccessConnectorIDs":[18134],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"metadata_stage_table_name","mapTo":"metadata_stage_table_name","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"source_metadata_sql","mapTo":"source_metadata_sql","validationStatus":null,"validationMessage":null},"3":{"slot":3,"fromId":null,"fromName":"get_metadata_sql","mapTo":"get_metadata_sql","validationStatus":null,"validationMessage":null},"4":{"slot":4,"fromId":null,"fromName":"stage_table_name","mapTo":"stage_table_name","validationStatus":null,"validationMessage":null},"5":{"slot":5,"fromId":null,"fromName":"target_table_name","mapTo":"target_table_name","validationStatus":null,"validationMessage":null}},"gridExportMappings":{"1":{"slot":1,"fromGrid":"source_list","toGrid":"source_list","columnMapToFrom":{"column_name":"column","key_column":"key_column","incremental_column":"incremental_column","table_name":"table"},"validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Build Strings"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"stage_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_database}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"stage_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"stage_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_project}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"stage_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"stage_prefix","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_prefix}"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"target_prefix","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_prefix}"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"metadata_stage_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"10":{"slot":10,"name":"stage_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"11":{"slot":11,"name":"target_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"12":{"slot":12,"name":"source_metadata_sql","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"13":{"slot":13,"name":"get_metadata_sql","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"14":{"slot":14,"name":"source_list","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"source_list"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"table_name"},"2":{"slot":2,"type":"GRID","value":"column_name"},"3":{"slot":3,"type":"GRID","value":"incremental_column"},"4":{"slot":4,"type":"GRID","value":"key_column"}}}},"visible":true,"status":{},"columnNames":[]},"15":{"slot":15,"name":"random_suffix","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"17586":{"id":17586,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1096987946,"x":160,"y":0,"width":32,"height":32,"inputConnectorIDs":[17597],"outputSuccessConnectorIDs":[17604],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"gridExportMappings":{"1":{"slot":1,"fromGrid":"source_table_metadata","toGrid":"source_table_metadata","columnMapToFrom":{"is_key":"is_key","nullable":"is_nullable","length":"length","column_name":"column_name","data_type_name":"data_type_name","numeric_scale":"numeric_scale","ordinal":"ordinal"},"validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Check Source Metadata"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"metadata_stage_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${metadata_stage_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"target_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"get_metadata_sql","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${get_metadata_sql}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"stage_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_database}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"stage_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"stage_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_project}"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"stage_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"source_table_metadata","elements":{},"visible":true,"status":{},"columnNames":[]},"10":{"slot":10,"name":"error_message","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"17587":{"id":17587,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-231597779,"x":480,"y":0,"width":32,"height":32,"inputConnectorIDs":[17604],"outputSuccessConnectorIDs":[17606],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"gridExportMappings":{"1":{"slot":1,"fromGrid":"required_columns","toGrid":"source_list","columnMapToFrom":{"column_name":"column_name","key_column":"key_column","incremental_column":"incremental_column","table_name":"table"},"validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Check Required Columns"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"target_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"actual_columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"source_table_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"column_name"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"required_columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"source_list"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"table_name"},"2":{"slot":2,"type":"GRID","value":"column_name"},"3":{"slot":3,"type":"GRID","value":"incremental_column"},"4":{"slot":4,"type":"GRID","value":"key_column"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"missing_columns","elements":{},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"error_message","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"17588":{"id":17588,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":947479920,"x":800,"y":0,"width":32,"height":32,"inputConnectorIDs":[17596],"outputSuccessConnectorIDs":[17287],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"incremental_column","mapTo":"incremental_column","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"incremental_column_count","mapTo":"incremental_column_count","validationStatus":null,"validationMessage":null},"3":{"slot":3,"fromId":null,"fromName":"SELECT_STRING","mapTo":"source_column_list","validationStatus":null,"validationMessage":null},"4":{"slot":4,"fromId":null,"fromName":"key_column_count","mapTo":"key_column_count","validationStatus":null,"validationMessage":null}},"gridExportMappings":{"1":{"slot":1,"fromGrid":"key_columns","toGrid":"key_columns","columnMapToFrom":{"column_name":"column_name"},"validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Parameter Parser"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"platform","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_platform}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"incremental_source_list","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"source_list"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"table_name"},"2":{"slot":2,"type":"GRID","value":"column_name"},"3":{"slot":3,"type":"GRID","value":"incremental_column"},"4":{"slot":4,"type":"GRID","value":"key_column"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"source_list","elements":{},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"stage_warehouse","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_warehouse}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"stage_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_database}"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"stage_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"stage_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_project}"}}}},"visible":true,"status":{},"columnNames":[]},"10":{"slot":10,"name":"stage_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"11":{"slot":11,"name":"stage_prefix","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_prefix}"}}}},"visible":true,"status":{},"columnNames":[]},"12":{"slot":12,"name":"target_warehouse","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_warehouse}"}}}},"visible":true,"status":{},"columnNames":[]},"13":{"slot":13,"name":"target_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"14":{"slot":14,"name":"target_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"15":{"slot":15,"name":"target_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_project}"}}}},"visible":true,"status":{},"columnNames":[]},"16":{"slot":16,"name":"target_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"17":{"slot":17,"name":"target_prefix","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_prefix}"}}}},"visible":true,"status":{},"columnNames":[]},"18":{"slot":18,"name":"maximum_incremental_columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"DECIMAL","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"19":{"slot":19,"name":"source_enclosing_char_e","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"\""}}}},"visible":true,"status":{},"columnNames":[]},"20":{"slot":20,"name":"source_enclosing_char_s","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"\""}}}},"visible":true,"status":{},"columnNames":[]},"21":{"slot":21,"name":"metadata_stage_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${metadata_stage_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"22":{"slot":22,"name":"stage_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"23":{"slot":23,"name":"target_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"24":{"slot":24,"name":"column_count","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"DECIMAL","value":"0"}}}},"visible":true,"status":{},"columnNames":[]},"25":{"slot":25,"name":"key_column_count","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"DECIMAL","value":"0"}}}},"visible":true,"status":{},"columnNames":[]},"26":{"slot":26,"name":"key_columns","elements":{},"visible":true,"status":{},"columnNames":[]},"27":{"slot":27,"name":"incremental_column_count","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"DECIMAL","value":"0"}}}},"visible":true,"status":{},"columnNames":[]},"28":{"slot":28,"name":"incremental_column","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"29":{"slot":29,"name":"SELECT_STRING","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"30":{"slot":30,"name":"stage_table_metadata_sql","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"17590":{"id":17590,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":960,"y":0,"width":32,"height":32,"inputConnectorIDs":[17287],"outputSuccessConnectorIDs":[17601],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Stage Single Row"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Facebook Content Insights - 4 - Query Source"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"warehouse"},"2":{"slot":2,"type":"STRING","value":"${stage_warehouse}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"database"},"2":{"slot":2,"type":"STRING","value":"${stage_database}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"schema"},"2":{"slot":2,"type":"STRING","value":"${stage_schema}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"project"},"2":{"slot":2,"type":"STRING","value":"${stage_project}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"dataset"},"2":{"slot":2,"type":"STRING","value":"${stage_dataset}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"table"},"2":{"slot":2,"type":"STRING","value":"${stage_table_name}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"s3_bucket_name"},"2":{"slot":2,"type":"STRING","value":"${s3_bucket_name}"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"dist_style"},"2":{"slot":2,"type":"STRING","value":"${dist_style}"}}},"9":{"slot":9,"values":{"1":{"slot":1,"type":"STRING","value":"staging_type"},"2":{"slot":2,"type":"STRING","value":"${staging_type}"}}},"10":{"slot":10,"values":{"1":{"slot":1,"type":"STRING","value":"blob_container"},"2":{"slot":2,"type":"STRING","value":"${blob_container}"}}},"11":{"slot":11,"values":{"1":{"slot":1,"type":"STRING","value":"storage_account"},"2":{"slot":2,"type":"STRING","value":"${storage_account}"}}},"12":{"slot":12,"values":{"1":{"slot":1,"type":"STRING","value":"cloud_storage_area"},"2":{"slot":2,"type":"STRING","value":"${cloud_storage_area}"}}},"13":{"slot":13,"values":{"1":{"slot":1,"type":"STRING","value":"encryption_method"},"2":{"slot":2,"type":"STRING","value":"${encryption_method}"}}},"14":{"slot":14,"values":{"1":{"slot":1,"type":"STRING","value":"kms_key_id"},"2":{"slot":2,"type":"STRING","value":"${kms_key_id}"}}},"15":{"slot":15,"values":{"1":{"slot":1,"type":"STRING","value":"auto_debug"},"2":{"slot":2,"type":"STRING","value":"${auto_debug}"}}},"16":{"slot":16,"values":{"1":{"slot":1,"type":"STRING","value":"auto_debug_level"},"2":{"slot":2,"type":"STRING","value":"${auto_debug_level}"}}},"17":{"slot":17,"values":{"1":{"slot":1,"type":"STRING","value":"target_table_name"},"2":{"slot":2,"type":"STRING","value":"${target_table_name}"}}},"18":{"slot":18,"values":{"1":{"slot":1,"type":"STRING","value":"oauth"},"2":{"slot":2,"type":"STRING","value":"${oauth}"}}},"19":{"slot":19,"values":{"1":{"slot":1,"type":"STRING","value":"key_value"},"2":{"slot":2,"type":"STRING","value":"${key_value}"}}},"20":{"slot":20,"values":{"1":{"slot":1,"type":"STRING","value":"period"},"2":{"slot":2,"type":"STRING","value":"${period}"}}},"21":{"slot":21,"values":{"1":{"slot":1,"type":"STRING","value":"limit"},"2":{"slot":2,"type":"STRING","value":"1"}}},"22":{"slot":22,"values":{"1":{"slot":1,"type":"STRING","value":"table_name"},"2":{"slot":2,"type":"STRING","value":"${table_name}"}}},"23":{"slot":23,"values":{"1":{"slot":1,"type":"STRING","value":"source_column_list"},"2":{"slot":2,"type":"STRING","value":"${source_column_list}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"advanced_connection_options"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"parameter"},"2":{"slot":2,"type":"GRID","value":"value"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"advanced_connection_options"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Facebook Content Insights Incremental Load"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Incremental"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"2"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"17591":{"id":17591,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":332733136,"x":1120,"y":0,"width":32,"height":32,"inputConnectorIDs":[17601],"outputSuccessConnectorIDs":[19239],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"load_strategy","mapTo":"load_strategy","validationStatus":null,"validationMessage":null},"2":{"slot":2,"fromId":null,"fromName":"max_value","mapTo":"max_value","validationStatus":null,"validationMessage":null}},"gridExportMappings":{"1":{"slot":1,"fromGrid":"stage_table_metadata","toGrid":"stage_table_metadata","columnMapToFrom":{"column_name":"column_name","database_type":"database_type","size":"size","type":"type","precision":"precision"},"validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Decide Load Strategy"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"source_column_list","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${source_column_list}"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"key_column_count","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${key_column_count}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"incremental_column_count","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${incremental_column_count}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"incremental_column","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${incremental_column}"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"skip_inc_col_check","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Y"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"timestamp_precision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${timestamp_precision}"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"safely_cast_datetime_to_date","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"N"}}}},"visible":true,"status":{},"columnNames":[]},"10":{"slot":10,"name":"encloseTableString","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"TRUE"}}}},"visible":true,"status":{},"columnNames":[]},"11":{"slot":11,"name":"source_enclosing_char_s","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"\""}}}},"visible":true,"status":{},"columnNames":[]},"12":{"slot":12,"name":"source_enclosing_char_e","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"\""}}}},"visible":true,"status":{},"columnNames":[]},"13":{"slot":13,"name":"target_platform","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_platform}"}}}},"visible":true,"status":{},"columnNames":[]},"14":{"slot":14,"name":"stage_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_database}"}}}},"visible":true,"status":{},"columnNames":[]},"15":{"slot":15,"name":"stage_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"16":{"slot":16,"name":"stage_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_project}"}}}},"visible":true,"status":{},"columnNames":[]},"17":{"slot":17,"name":"stage_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"18":{"slot":18,"name":"stage_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${stage_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"19":{"slot":19,"name":"target_database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_database}"}}}},"visible":true,"status":{},"columnNames":[]},"20":{"slot":20,"name":"target_schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_schema}"}}}},"visible":true,"status":{},"columnNames":[]},"21":{"slot":21,"name":"target_project","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_project}"}}}},"visible":true,"status":{},"columnNames":[]},"22":{"slot":22,"name":"target_dataset","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_dataset}"}}}},"visible":true,"status":{},"columnNames":[]},"23":{"slot":23,"name":"target_table_name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${target_table_name}"}}}},"visible":true,"status":{},"columnNames":[]},"24":{"slot":24,"name":"fix_target_data_types","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Yes"}}}},"visible":true,"status":{},"columnNames":[]},"25":{"slot":25,"name":"load_strategy","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"26":{"slot":26,"name":"max_value","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"27":{"slot":27,"name":"stage_sql","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"28":{"slot":28,"name":"inc_col_type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":""}}}},"visible":true,"status":{},"columnNames":[]},"29":{"slot":29,"name":"stage_table_metadata","elements":{},"visible":true,"status":{},"columnNames":[]},"30":{"slot":30,"name":"messages","elements":{},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"17594":{"id":17594,"inputCardinality":"ONE","outputCardinality":"ZERO","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":-1946388514,"x":1920,"y":0,"width":32,"height":32,"inputConnectorIDs":[16227],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"End Success"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"17595":{"id":17595,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186960,"x":640,"y":0,"width":32,"height":32,"inputConnectorIDs":[17606],"outputSuccessConnectorIDs":[17596],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Adjust Source List"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"def truncate(word, length = 12):\n \"\"\"\n `truncate()` will convert items to a string and ensure they are the length specified by user \n Used within stringer function to ensure all columns are the same width \n \"\"\"\n if len(word.strip()) <= length:\n return word\n return \"{}...\".format( word[:(length-3)] )\n \ndef stringer(lst, colwidth = 30):\n \"\"\"\n `stringer()` is used for printing Grid Variables in a more readable manner.\n \n It will handle formatting the grid variable to look more like how it is presented in the product \n (as a table) \n \"\"\"\n msgs = [] \n for row in lst:\n msg = \" | \".join(\n \t[\n \"{:<{width}}\".format( truncate(str(x), length = colwidth), width = colwidth)\n for x in row \n ]\n )\n msgs.append(msg)\n \n return \"\\n\" + \"\\n\".join(msgs)\n\nsource_list = context.getGridVariable('source_list')\nprint(\"Grid variable 'source_list': {}\".format(stringer(source_list)))\n \nsource_table_metadata = context.getGridVariable('source_table_metadata')\nprint(\"\\nGrid variable 'source_table_metadata': {}\".format(stringer(source_table_metadata)))\n\ninc_col_options = ['FromDateTime']\nprint(\"\\nVariable 'inc_col_options': {}\".format(inc_col_options))\n\n# is_table() checks that a row of a grid variable belongs to the table we \n# are currently iterating over\nis_table = lambda x: x[0] == table_name\n\n# filter source_list to current table and extract list of columns\nsrc = [ x for x in source_list if is_table(x) ] \nrequested_columns = [ x[1] for x in src ]\n\n# extract key and inc columns\n# Note that the true/false values for IsKey are sometimes truncated to three letters, i.e. tru/fal\n#key_columns = [ x[0] for x in source_table_metadata if x[6].lower().startswith('t')]\nkey_columns = ['Target', 'InsightName', 'EndTime' ]\n\n# loop through ordered list of possible incremental columns, using the first one found\nmetadata_columns = [ x[0] for x in source_table_metadata ]\ninc_columns = []\nfor i in inc_col_options:\n if i in metadata_columns:\n inc_columns = [i]\n break\n\n# add missing columns, preserving order set by user\nall_columns = requested_columns\n\n\nfor x in key_columns:\n if x not in all_columns:\n all_columns.append(x)\n\nfor x in inc_columns:\n if x not in all_columns:\n all_columns.append(x) \n \n# we include the ToDateTime pseudo column for convenience - this is not used as an incremental column however \nfor x in ['ToDateTime']:\n if x not in all_columns:\n all_columns.append(x) \n \n# build new source column list with inc and key col flags\nnew_src = [] \n\nfor c in all_columns:\n new_src.append(\n [\n table_name, \n c,\n '1' if c in inc_columns else '0',\n '1' if c in key_columns else '0'\n \t\t]\n \t)\n \ncontext.updateGridVariable('source_list', new_src)\nprint(\"\\nUpdated grid variable 'source_list': {}\".format(stringer(new_src)))\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"18105":{"id":18105,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186829,"x":-160,"y":0,"width":32,"height":32,"inputConnectorIDs":[18134],"outputSuccessConnectorIDs":[18137],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Instagram Business Account ID"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"if table_name.lower() == 'instagramaccountinsights':\n \n if not instagram_business_acount_id:\n raise Exception(\"InstagramAccountInsights requested but no Instagram Business Account ID supplied\") \n else:\n context.updateValue('key_value', instagram_business_account_id)\n \nprint(f\"Table Name: {table_name} | Key Value: {key_value} | Initial Start Date: {initial_start_date} | Period: {period}\")"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"User","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Restricted"}}}},"visible":false,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"19231":{"id":19231,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"EXECUTE","implementationID":-1773186960,"x":1440,"y":0,"width":32,"height":32,"inputConnectorIDs":[19239],"outputSuccessConnectorIDs":[19257],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Set Staging Vars"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Script","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"from datetime import datetime, timedelta\n\nbatch_size = 30\n\n_dateformat_start = \"%Y-%m-%d\"\n_dateformat_end = \"%Y-%m-%d\"\n\n# We need to add 2 days to each filter when sending requests to Facebook \nend_date = (datetime.now() + timedelta(days=2)).strftime(_dateformat_end) \ncontext.updateVariable('end_date', end_date) \n\n# max_value will only be set on subsequent loads as the target table (which it is derived from) does not exist yet. \nif not max_value:\n print(f\"Using initial start date as starting date: {initial_start_date}\")\n start_date = initial_start_date \nelse:\n # max value is stored as a datetime in the database\n max_value_date = datetime.strptime(max_value, '%Y-%m-%d %H:%M:%S')\n print(f\"Using max_value as starting date: {max_value_date}\")\n start_date = max_value_date.strftime(_dateformat_start) \n \ncontext.updateVariable('start_date', start_date) \n\nstart_dt = datetime.strptime(start_date, _dateformat_start)\nend_dt = datetime.strptime(end_date, _dateformat_end) \n\nfrom math import ceil \n\nexpected_batches = ceil((end_dt - start_dt).days/batch_size)\n\nintervals = [\n ## each interval grabs data for 30 days \n ## there are some qwirks with the Facebook API and it only returns data for 2 days prior to the filter ToDateTime\n ## so we need to overlap each of the intervals by 2 days to make sure no data is missed \n ## we hardcode key columns in a previous script (Adjust Source List) meaning we do not need to worry about duplicated data.\n\tstart_dt + timedelta(days = n*batch_size + 2) for n in range(0, expected_batches)\n]\n\n\nif len(intervals) == 0:\n intervals = [start_dt + timedelta(days = 2)]\n\ntotal = len(intervals)\n\niter_list = [\n\t[\n # first batch should start from the start date \n x if n > 0 else start_dt, \n\n # last batch should end with the end date\n x + timedelta(days = batch_size + 2) if n + 1 < total else end_dt, \n \n # only want to truncate insert the first batch on the first run \n 'TRUNCATE INSERT' if (n==0 and load_strategy == 'TRUNCATE INSERT') else 'UPDATE' \n ] \n for n, x in enumerate(intervals)\n]\n\nprint(f\"Generated (`staging_vars` Grid Variable) {len(iter_list)} batches starting from {start_dt} and ending at {end_dt}:\\n\\n\")\nfor r in iter_list:\n print(f\"Start = {r[0]} | End = {r[1]} | Strategy = {r[2]}\\n\")\n \ncontext.updateGridVariable('staging_vars', iter_list)\n"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Interpreter","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Python 3"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Timeout","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"360"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"19256":{"id":19256,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"ITERATE","implementationID":-424773870,"x":1600,"y":0,"width":32,"height":16,"inputConnectorIDs":[19257],"outputSuccessConnectorIDs":[16227],"outputFailureConnectorIDs":[13531],"outputUnconditionalConnectorIDs":[16231],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Iterate Over Range"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Grid Variable","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"staging_vars"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Grid Variable Mapping","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"end"},"2":{"slot":2,"type":"STRING","value":"iter_end_date"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"start"},"2":{"slot":2,"type":"STRING","value":"iter_start_date"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"load_strategy"},"2":{"slot":2,"type":"STRING","value":"iter_load_strategy"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Break on Failure","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"No"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Concurrency","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Sequential"}}}},"visible":true,"status":{},"columnNames":[]},"999":{"slot":999,"name":"Record Values In Task History","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Yes"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[19264],"inputIterationConnectorIDs":[]},"19261":{"id":19261,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-1180878087,"x":1600,"y":32,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{"1":{"slot":1,"fromId":null,"fromName":"logged_rows_inserted","mapTo":"logged_rows_inserted","validationStatus":null,"validationMessage":null}},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Load Content Insights"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Facebook Content Insights - 3a - Date"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"dist_style"},"2":{"slot":2,"type":"STRING","value":"${dist_style}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"table_name"},"2":{"slot":2,"type":"STRING","value":"${table_name}"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"stage_schema"},"2":{"slot":2,"type":"STRING","value":"${stage_schema}"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"target_schema"},"2":{"slot":2,"type":"STRING","value":"${target_schema}"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"s3_bucket_name"},"2":{"slot":2,"type":"STRING","value":"${s3_bucket_name}"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"stage_table_name"},"2":{"slot":2,"type":"STRING","value":"${stage_table_name}"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"target_table_name"},"2":{"slot":2,"type":"STRING","value":"${target_table_name}"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"target_platform"},"2":{"slot":2,"type":"STRING","value":"${target_platform}"}}},"9":{"slot":9,"values":{"1":{"slot":1,"type":"STRING","value":"target_database"},"2":{"slot":2,"type":"STRING","value":"${target_database}"}}},"10":{"slot":10,"values":{"1":{"slot":1,"type":"STRING","value":"target_warehouse"},"2":{"slot":2,"type":"STRING","value":"${target_warehouse}"}}},"11":{"slot":11,"values":{"1":{"slot":1,"type":"STRING","value":"stage_database"},"2":{"slot":2,"type":"STRING","value":"${stage_database}"}}},"12":{"slot":12,"values":{"1":{"slot":1,"type":"STRING","value":"stage_warehouse"},"2":{"slot":2,"type":"STRING","value":"${stage_warehouse}"}}},"13":{"slot":13,"values":{"1":{"slot":1,"type":"STRING","value":"target_dataset"},"2":{"slot":2,"type":"STRING","value":"${target_dataset}"}}},"14":{"slot":14,"values":{"1":{"slot":1,"type":"STRING","value":"target_project"},"2":{"slot":2,"type":"STRING","value":"${target_project}"}}},"15":{"slot":15,"values":{"1":{"slot":1,"type":"STRING","value":"stage_dataset"},"2":{"slot":2,"type":"STRING","value":"${stage_dataset}"}}},"16":{"slot":16,"values":{"1":{"slot":1,"type":"STRING","value":"stage_project"},"2":{"slot":2,"type":"STRING","value":"${stage_project}"}}},"17":{"slot":17,"values":{"1":{"slot":1,"type":"STRING","value":"cloud_storage_area"},"2":{"slot":2,"type":"STRING","value":"${cloud_storage_area}"}}},"18":{"slot":18,"values":{"1":{"slot":1,"type":"STRING","value":"storage_account"},"2":{"slot":2,"type":"STRING","value":"${storage_account}"}}},"19":{"slot":19,"values":{"1":{"slot":1,"type":"STRING","value":"staging_type"},"2":{"slot":2,"type":"STRING","value":"${staging_type}"}}},"20":{"slot":20,"values":{"1":{"slot":1,"type":"STRING","value":"blob_container"},"2":{"slot":2,"type":"STRING","value":"${blob_container}"}}},"21":{"slot":21,"values":{"1":{"slot":1,"type":"STRING","value":"encryption_method"},"2":{"slot":2,"type":"STRING","value":"${encryption_method}"}}},"22":{"slot":22,"values":{"1":{"slot":1,"type":"STRING","value":"kms_key_id"},"2":{"slot":2,"type":"STRING","value":"${kms_key_id}"}}},"23":{"slot":23,"values":{"1":{"slot":1,"type":"STRING","value":"incremental_column"},"2":{"slot":2,"type":"STRING","value":"${incremental_column}"}}},"24":{"slot":24,"values":{"1":{"slot":1,"type":"STRING","value":"logged_rows_inserted"},"2":{"slot":2,"type":"STRING","value":"${logged_rows_inserted}"}}},"25":{"slot":25,"values":{"1":{"slot":1,"type":"STRING","value":"log_metrics"},"2":{"slot":2,"type":"STRING","value":"${log_metrics}"}}},"26":{"slot":26,"values":{"1":{"slot":1,"type":"STRING","value":"auto_debug"},"2":{"slot":2,"type":"STRING","value":"${auto_debug}"}}},"27":{"slot":27,"values":{"1":{"slot":1,"type":"STRING","value":"auto_debug_level"},"2":{"slot":2,"type":"STRING","value":"${auto_debug_level}"}}},"28":{"slot":28,"values":{"1":{"slot":1,"type":"STRING","value":"source_column_list"},"2":{"slot":2,"type":"STRING","value":"${source_column_list}"}}},"29":{"slot":29,"values":{"1":{"slot":1,"type":"STRING","value":"load_strategy"},"2":{"slot":2,"type":"STRING","value":"${iter_load_strategy}"}}},"30":{"slot":30,"values":{"1":{"slot":1,"type":"STRING","value":"start_date"},"2":{"slot":2,"type":"STRING","value":"${iter_start_date}"}}},"31":{"slot":31,"values":{"1":{"slot":1,"type":"STRING","value":"end_date"},"2":{"slot":2,"type":"STRING","value":"${iter_end_date}"}}},"32":{"slot":32,"values":{"1":{"slot":1,"type":"STRING","value":"oauth"},"2":{"slot":2,"type":"STRING","value":"${oauth}"}}},"33":{"slot":33,"values":{"1":{"slot":1,"type":"STRING","value":"period"},"2":{"slot":2,"type":"STRING","value":"${period}"}}},"34":{"slot":34,"values":{"1":{"slot":1,"type":"STRING","value":"key_value"},"2":{"slot":2,"type":"STRING","value":"${key_value}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{"1":{"slot":1,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"advanced_connection_options"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"parameter"},"2":{"slot":2,"type":"GRID","value":"value"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"advanced_connection_options"},"2":{"slot":2,"type":"STRING","value":"grid"}}},"2":{"slot":2,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"key_columns"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"column_name"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"key_columns"},"2":{"slot":2,"type":"STRING","value":"grid"}}},"3":{"slot":3,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"source_list"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"table_name"},"2":{"slot":2,"type":"GRID","value":"column_name"},"3":{"slot":3,"type":"GRID","value":"incremental_column"},"4":{"slot":4,"type":"GRID","value":"key_column"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"source_list"},"2":{"slot":2,"type":"STRING","value":"grid"}}},"4":{"slot":4,"elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"GRID","value":"stage_table_metadata"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"GRID","value":"column_name"},"2":{"slot":2,"type":"GRID","value":"type"},"3":{"slot":3,"type":"GRID","value":"size"},"4":{"slot":4,"type":"GRID","value":"precision"},"5":{"slot":5,"type":"GRID","value":"database_type"}}}},"values":{"1":{"slot":1,"type":"STRING","value":"stage_table_metadata"},"2":{"slot":2,"type":"STRING","value":"grid"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Facebook Content Insights Incremental Load"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion.Incremental"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"2"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[19264]}},"successConnectors":{"16227":{"id":16227,"sourceID":19256,"targetID":17594},"17287":{"id":17287,"sourceID":17588,"targetID":17590},"17596":{"id":17596,"sourceID":17595,"targetID":17588},"17597":{"id":17597,"sourceID":17584,"targetID":17586},"17601":{"id":17601,"sourceID":17590,"targetID":17591},"17604":{"id":17604,"sourceID":17586,"targetID":17587},"17606":{"id":17606,"sourceID":17587,"targetID":17595},"18134":{"id":18134,"sourceID":17585,"targetID":18105},"18137":{"id":18137,"sourceID":18105,"targetID":17584},"19239":{"id":19239,"sourceID":17591,"targetID":19231},"19257":{"id":19257,"sourceID":19231,"targetID":19256}},"failureConnectors":{"13531":{"id":13531,"sourceID":19256,"targetID":16233}},"unconditionalConnectors":{"16231":{"id":16231,"sourceID":19256,"targetID":16215},"18140":{"id":18140,"sourceID":17583,"targetID":17585}},"trueConnectors":{"16274":{"id":16274,"sourceID":16215,"targetID":16267}},"falseConnectors":{},"iterationConnectors":{"19264":{"id":19264,"sourceID":19256,"targetID":19261}},"noteConnectors":{},"canUndo":false,"undoCommand":"","undoCreated":0,"canRedo":false,"redoCommand":"","redoCreated":0,"notes":{"17579":{"id":17579,"x":-394,"y":-233,"width":629,"height":312,"text":"** Source Validation **\n\n- (__Build Strings__) Generate generic job variables (stage/target table names etc. - see Export tab) \n\n- (__Instagram Business Account ID__) If InstagramAccountInsights, ensures that an Instagram Business Account ID is supplied\n\n- (__Get Source Metadata__) Query CData __sys_tablecolumns__ metadata table to get information about the source and store in CDW \n\n- (__Check Source Metadata__) Query table created above, ensure it is valid, export the metadata to a grid (see Grid Export tab)","colour":"e6e63c"},"17580":{"id":17580,"x":406,"y":-316,"width":789,"height":396,"text":"**Input/Target Validation**\n\n- (__Check Required Columns__) Contains logic to validate that the columns which have been requested still exist in the source\n\n- (__Adjust Source List__) Modifies the current __source_list__ grid variable. This will set the correct key and incremental columns (where applicable). \nNB: For Facebook Content Insights, the __'FromDateTime'__ pseudo-column is used as an incremental column. \nA composite key of Target, InsightName, EndTime is used for all tables. \nThe 'ToDateTime' pseudo column is also always included for convenience (as neither a key or an incremental column) \n\n- (__Parameter Parser__) Contains logic to generate some useful Grid/Job variables (e.g key_columns Grid, SELECT list. See Export/Grid Export tabs)\n\n- (__Stage Single Row__) Stage 1 row from the data source.\n\n- (__Decide Load Strategy__) Executes packaged up logic to determine the type of load this job is doing (UPDATE/TRUNCATE INSERT). \n**Note: For Facebook Content Insights, we create send batches of requests for different timeframes, so the Load Strategy is only TRUNCATE INSERT for the first batch of the first run, any subsequent loads will perform an UPDATE**","colour":"e6e63c"},"17582":{"id":17582,"x":1364,"y":-240,"width":471,"height":314,"text":"** Load Data **\n\n- (__Set Staging Vars__) Script generates a list of date ranges (or batches) or at most 30 days and exports this into the __staging_vars__ Grid Variable.\n\n- (__Iterate Over Range__) Iterates over the __staging_vars__ Grid Variable and executes the stage and load logic for each set of parameters.\n\n**Note: The Facebook API seems to only return data from 2 days prior to the ToDateTime filter, which means we must bump each day by 2 and overlap the Date Batches**","colour":"e6e63c"}},"variables":{"oauth":{"definition":{"name":"oauth","type":"TEXT","scope":"TASKBATCH","description":"The Matillion OAuth name to use when connecting to Sales Force.com","visibility":"PUBLIC"},"value":""},"period":{"definition":{"name":"period","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"end_date":{"definition":{"name":"end_date","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":null},"key_value":{"definition":{"name":"key_value","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"max_value":{"definition":{"name":"max_value","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"stage_sql":{"definition":{"name":"stage_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":null},"auto_debug":{"definition":{"name":"auto_debug","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"dist_style":{"definition":{"name":"dist_style","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"kms_key_id":{"definition":{"name":"kms_key_id","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"start_date":{"definition":{"name":"start_date","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":null},"table_name":{"definition":{"name":"table_name","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"log_metrics":{"definition":{"name":"log_metrics","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_prefix":{"definition":{"name":"stage_prefix","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_schema":{"definition":{"name":"stage_schema","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"staging_type":{"definition":{"name":"staging_type","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"error_message":{"definition":{"name":"error_message","type":"TEXT","scope":"BRANCH","description":"","visibility":"PRIVATE"},"value":""},"iter_end_date":{"definition":{"name":"iter_end_date","type":"TEXT","scope":"BRANCH","description":"","visibility":"PRIVATE"},"value":""},"load_strategy":{"definition":{"name":"load_strategy","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"TRUNCATE INSERT"},"stage_dataset":{"definition":{"name":"stage_dataset","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_project":{"definition":{"name":"stage_project","type":"TEXT","scope":"BRANCH","description":" ","visibility":"PUBLIC"},"value":""},"target_prefix":{"definition":{"name":"target_prefix","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_schema":{"definition":{"name":"target_schema","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"blob_container":{"definition":{"name":"blob_container","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"s3_bucket_name":{"definition":{"name":"s3_bucket_name","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"stage_database":{"definition":{"name":"stage_database","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_dataset":{"definition":{"name":"target_dataset","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_project":{"definition":{"name":"target_project","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"iter_start_date":{"definition":{"name":"iter_start_date","type":"TEXT","scope":"BRANCH","description":"","visibility":"PRIVATE"},"value":""},"stage_warehouse":{"definition":{"name":"stage_warehouse","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"storage_account":{"definition":{"name":"storage_account","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_database":{"definition":{"name":"target_database","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_platform":{"definition":{"name":"target_platform","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"auto_debug_level":{"definition":{"name":"auto_debug_level","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"get_metadata_sql":{"definition":{"name":"get_metadata_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"key_column_count":{"definition":{"name":"key_column_count","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"stage_single_sql":{"definition":{"name":"stage_single_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"stage_table_name":{"definition":{"name":"stage_table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"static_row_count":{"definition":{"name":"static_row_count","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"target_warehouse":{"definition":{"name":"target_warehouse","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"encryption_method":{"definition":{"name":"encryption_method","type":"TEXT","scope":"BRANCH","description":"","visibility":"PUBLIC"},"value":""},"target_table_name":{"definition":{"name":"target_table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"cloud_storage_area":{"definition":{"name":"cloud_storage_area","type":"TEXT","scope":"BRANCH","description":"The cloud storage area name that should be used when loading bq tables.","visibility":"PUBLIC"},"value":""},"incremental_column":{"definition":{"name":"incremental_column","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"initial_start_date":{"definition":{"name":"initial_start_date","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"2021-03-19"},"iter_load_strategy":{"definition":{"name":"iter_load_strategy","type":"TEXT","scope":"BRANCH","description":"","visibility":"PRIVATE"},"value":""},"source_column_list":{"definition":{"name":"source_column_list","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"logged_rows_deleted":{"definition":{"name":"logged_rows_deleted","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"0"},"logged_rows_updated":{"definition":{"name":"logged_rows_updated","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"0"},"source_metadata_sql":{"definition":{"name":"source_metadata_sql","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":""},"timestamp_precision":{"definition":{"name":"timestamp_precision","type":"TEXT","scope":"BRANCH","description":"","visibility":"PRIVATE"},"value":"microseconds"},"logged_rows_inserted":{"definition":{"name":"logged_rows_inserted","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":"0"},"incremental_column_count":{"definition":{"name":"incremental_column_count","type":"DECIMAL","scope":"TASKBATCH","description":"","visibility":"PRIVATE"},"value":"0"},"metadata_stage_table_name":{"definition":{"name":"metadata_stage_table_name","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""},"instagram_business_account_id":{"definition":{"name":"instagram_business_account_id","type":"TEXT","scope":"TASKBATCH","description":"","visibility":"PUBLIC"},"value":""}},"grids":{"key_columns":{"definition":{"name":"key_columns","scope":"BRANCH","definitions":[{"name":"column_name","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[]},"source_list":{"definition":{"name":"source_list","scope":"BRANCH","definitions":[{"name":"table_name","type":"TEXT"},{"name":"column_name","type":"TEXT"},{"name":"incremental_column","type":"DECIMAL"},{"name":"key_column","type":"DECIMAL"}],"description":"","visibility":"PUBLIC"},"values":[]},"staging_vars":{"definition":{"name":"staging_vars","scope":"BRANCH","definitions":[{"name":"start","type":"TEXT"},{"name":"end","type":"TEXT"},{"name":"load_strategy","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[]},"stage_table_metadata":{"definition":{"name":"stage_table_metadata","scope":"BRANCH","definitions":[{"name":"column_name","type":"TEXT"},{"name":"type","type":"TEXT"},{"name":"size","type":"TEXT"},{"name":"precision","type":"TEXT"},{"name":"database_type","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[]},"source_table_metadata":{"definition":{"name":"source_table_metadata","scope":"BRANCH","definitions":[{"name":"column_name","type":"TEXT"},{"name":"data_type_name","type":"TEXT"},{"name":"length","type":"TEXT"},{"name":"numeric_scale","type":"TEXT"},{"name":"nullable","type":"TEXT"},{"name":"ordinal","type":"TEXT"},{"name":"is_key","type":"TEXT"}],"description":"","visibility":"PRIVATE"},"values":[]},"advanced_connection_options":{"definition":{"name":"advanced_connection_options","scope":"BRANCH","definitions":[{"name":"parameter","type":"TEXT"},{"name":"value","type":"TEXT"}],"description":"","visibility":"PUBLIC"},"values":[]}}}},"transformationJobs":{}}],"version":"Built","environment":"redshift"}