{"objects":[{"metadata":{"identifier":{"packageName":"Matillion Exchange","name":"Schema Table Profiler","revision":1,"type":"DYNAMIC"},"rootJobReference":{"name":"Schema Table Profiler Entry","type":"ORCHESTRATION","parameterMetadata":[{"slot":2,"variableName":"jv_database_name","variableType":"SCALAR","displayName":"Database Name","description":"Name of the database containing the schema to profile","defaultValue":[{"values":{"1":"${environment_database}"}}],"defaultValueType":"TEXT","required":true},{"slot":3,"variableName":"jv_schema_name","variableType":"SCALAR","displayName":"Schema Name","description":"Name of the schema to profile","defaultValue":[{"values":{"1":"${environment_default_schema}"}}],"defaultValueType":"TEXT","required":true}]},"orchestrationJobs":["Schema Table Profiler Entry"],"transformationJobs":["Schema Table Profiler Queries"],"imageMetadata":{"componentIcon":"iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAACsElEQVR42mP4P8CAYVA54NOnT/9PnjxBM/zs2TP8Dmhvb/svLiFGM+zt4/X/zZs32B3Q2NQIVmRhaQ5kN1AdFxUV/peTlwU7Ajkk4A7IzcsBO+DsubM0i+/q6iqwHaDowOmAu3fv0MwBK1etHFgHrF+/jrADbty4TjMHzJkzm7ADLK0s/kdGReDE1TXV4NyCDYdHhOHVK68gR9gB9MB4HZCekf5/6tSpGNjdww0sHxIa/P/gwYP/Dx8+jILXrlv7X0VVGawmMzMDqxmmZqbkJ8K7d++C5Q8fOYwzjru6u/7n5GT///v3L1b5S5cu/bezt/t/8dJF0h1w/8F9sPyxY0dxOgCXxUTVBdRwAEWVEbEO2Lt3D1b5P3/+gOPew8OdIL5y5QpuB5iamfx3dXPBwDq6OmB5dQ01uBgo2128eAFumH+APzylS0pJ/JeSlkTBEpLihBOhtIzUf1k5GQwMMhAkDzIEJqZvoAfMEQfght27d++/gaE+WN358+cwQmnBgvlgh+B1wNWrV7EG8f79+8DyS5cuxRunO3bsANd69g52/79//44h//Lly//fvn2jbSLs6GgHq80ApokByQWgBFlYVAAO7lXAGnBAsuGPHz/++/n5/ldQlAeXlEQ7ICk56f86YLG6d+9eFAwqokHyYeFh/zdt3oQi9/79e6yGT58xHawnMSmRsAPevn0LrLEiyapcpkydgmHw48eP/2tqafw3NDL4f+r0KeJaxT9//vx/+fLl/7t27yIJv3v3DsXQX79+/XdzdwXnhmPHj9G3XwCqD9LT08DlxqLFi+jfMWmDNu2rgA1QdPDly+f/M2bO+P/o0SPaOGAzMHGCfA4K+ufPn2PId/d04y6KKQWgvA9KcLCECWp+KSkroWC8dQGlAJTwQFmZGAxq3AzOzulAAACOkicODgS7KAAAAABJRU5ErkJggg==","smallIcon":"iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAABi0lEQVR42qWTy0sCURjF/UOqRQU9NKiooKLMHmpJZkG1qfY9LFPbZIsy3Zgt07CVTUKCe7fChC1cmw+afG4SEQRBcOHJuYubw4ANdOAM98y9/L7vfszI8E/J+Af3xcFkOpNsh9OBRqPxC9Bo1fA/+xGJRCTZfmsnEApYXFKhUqlIbjv+ESedCAC+Jx9CoRA1X4llWeJwOCzYc7aqiwA63Rp2dreJ+/p7sbllgPH0hHhBpYRcIaf788o5MaD9CjbbJfL5PM25fA7pdJrmWq0G9o3tALiyIZvN0pxKJcG8MAJH36NCgGJEjrHxUeLuni4MDg1gZnYaxWIRsViMvLu+uYb73g3rhRWmc5MQUC6XaUWL1SxomZf30duah5Hmer3e+QrtM+DVbDZJZYZhxF+ixWKGwbCBo+NDYrliGHr9Oll7PA/0cDD4iv2DPTGApxcKBXDcp8il0jc5mEgkoF3VoFqtigF/iZ/P5NQEkskkyZlMBh6vRzogEAiQOfH/jFarwYp6Ga47l3RAJ/0AIhNJv502KyEAAAAASUVORK5CYII="},"description":"Automated table level data profiling for all tables in a named schema","helpHtml":"

Schema Table Profiler

Automated table level data profiling for all tables in a named schema

Properties

PropertyTypeDescription
Database NameVariableName of the database containing the schema to profile
Schema NameVariableName of the schema to profile
","created":1700580041243},"orchestrationJobs":{"Schema Table Profiler Entry":{"id":-2,"revision":16,"created":1700578854370,"timestamp":1700578854370,"components":{"10204":{"id":10204,"inputCardinality":"ZERO","outputCardinality":"MANY","connectorHint":"UNCONDITIONAL","executionHint":"FLOW","implementationID":444132438,"x":-352,"y":32,"width":32,"height":32,"inputConnectorIDs":[],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[10292],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Start","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Start 0"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]},"10291":{"id":10291,"inputCardinality":"ONE","outputCardinality":"MANY","connectorHint":"SUCCESS_FAIL","executionHint":"TRANSFORM","implementationID":-193159521,"x":-176,"y":34,"width":32,"height":32,"inputConnectorIDs":[10292],"outputSuccessConnectorIDs":[],"outputFailureConnectorIDs":[],"outputUnconditionalConnectorIDs":[],"outputTrueConnectorIDs":[],"outputFalseConnectorIDs":[],"exportMappings":{},"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Profile all tables in schema"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Job Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Schema Table Profiler Queries"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"","elements":{},"visible":false,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Set Scalar Variables","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"jv_database_name"},"2":{"slot":2,"type":"STRING","value":"${jv_database_name}"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"jv_schema_name"},"2":{"slot":2,"type":"STRING","value":"${jv_schema_name}"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Set Grid Variables","elements":{},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Melt Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Schema Table Profiler"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"Melt Package","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Matillion Exchange"}}}},"visible":true,"status":{},"columnNames":[]},"8":{"slot":8,"name":"Melt Revision","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"INTEGER","value":"1"}}}},"visible":true,"status":{},"columnNames":[]},"9":{"slot":9,"name":"Melt Load Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"DYNAMIC"}}}},"visible":true,"status":{},"columnNames":[]}},"validationStatus":{},"status":{},"expectedFailure":null,"activationStatus":"ENABLED","outputIterationConnectorIDs":[],"inputIterationConnectorIDs":[]}},"successConnectors":{},"failureConnectors":{},"unconditionalConnectors":{"10292":{"id":10292,"sourceID":10204,"targetID":10291}},"trueConnectors":{},"falseConnectors":{},"iterationConnectors":{},"noteConnectors":{},"canUndo":true,"undoCommand":"Delete Job Grid Variable","undoCreated":1700578762015,"canRedo":false,"redoCommand":"","redoCreated":-1,"notes":{},"variables":{"jv_schema_name":{"definition":{"name":"jv_schema_name","type":"TEXT","scope":"TASKBATCH","description":"Name of the schema to profile","visibility":"PUBLIC"},"value":"${environment_default_schema}"},"jv_database_name":{"definition":{"name":"jv_database_name","type":"TEXT","scope":"TASKBATCH","description":"Name of the database containing the schema to profile","visibility":"PUBLIC"},"value":"${environment_database}"}},"grids":{}}},"transformationJobs":{"Schema Table Profiler Queries":{"id":-2,"revision":16,"created":1700579577768,"timestamp":1700579577768,"components":{"10322":{"id":10322,"inputCardinality":"ZERO_OR_MANY","outputCardinality":"MANY","implementationID":-1266674941,"x":634,"y":259,"width":32,"height":32,"inputConnectorIDs":[],"outputConnectorIDs":[10330],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Table Profile"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"SQL Query","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"SELECT TABLE_NAME,\n TABLE_CATALOG,\n TABLE_SCHEMA,\n TABLE_OWNER,\n TABLE_TYPE,\n IS_TRANSIENT,CLUSTERING_KEY,\n ROW_COUNT,\n BYTES,\n ROUND(((BYTES/1024)/1024),2) AS SUM_BYTES_IN_MB,\n RETENTION_TIME,\n CREATED,\n LAST_ALTERED,\n rank() over (order by last_altered asc) as LAST_ALTERED_RANK,\n AUTO_CLUSTERING_ON,\n COMMENT\nFROM \"${jv_database_name}\".INFORMATION_SCHEMA.TABLES\nWHERE TABLE_SCHEMA = '${jv_schema_name}'\nAND TABLE_TYPE = 'BASE TABLE'"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"10323":{"id":10323,"inputCardinality":"ZERO_OR_MANY","outputCardinality":"MANY","implementationID":-1266674941,"x":634,"y":451,"width":32,"height":32,"inputConnectorIDs":[],"outputConnectorIDs":[10328],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Schema Profile"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"SQL Query","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"SELECT TABLE_SCHEMA,\n ROUND(SUM((BYTES/1024/1024)),2) AS SCHEMA_TOTAL_BYTES_IN_MB,\n MAX(LAST_ALTERED) as SCHEMA_LATEST_ALTERED_TABLE,\n MIN(LAST_ALTERED) AS SCHEMA_OLDEST_ALTERED_TABLE \nFROM \"${jv_database_name}\".INFORMATION_SCHEMA.TABLES \nWHERE TABLE_SCHEMA = '${jv_schema_name}'\nGROUP BY TABLE_SCHEMA"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"10324":{"id":10324,"inputCardinality":"MANY","outputCardinality":"MANY","implementationID":-629958239,"x":842,"y":355,"width":32,"height":32,"inputConnectorIDs":[10328,10330],"outputConnectorIDs":[10329],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Join Schema and Table Metadata"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Main Table","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Table Profile"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Main Table Alias","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"main"}}}},"visible":true,"status":{},"columnNames":[]},"4":{"slot":4,"name":"Joins","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Schema Profile"},"2":{"slot":2,"type":"STRING","value":"join"},"3":{"slot":3,"type":"STRING","value":"Inner"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Join Expressions","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"\"main\".\"TABLE_SCHEMA\" = \"join\".\"TABLE_SCHEMA\""},"2":{"slot":2,"type":"STRING","value":"main_Inner_join"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Output Columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"main.TABLE_NAME"},"2":{"slot":2,"type":"STRING","value":"TABLE_NAME"}}},"2":{"slot":2,"values":{"1":{"slot":1,"type":"STRING","value":"main.TABLE_CATALOG"},"2":{"slot":2,"type":"STRING","value":"TABLE_CATALOG"}}},"3":{"slot":3,"values":{"1":{"slot":1,"type":"STRING","value":"main.TABLE_SCHEMA"},"2":{"slot":2,"type":"STRING","value":"TABLE_SCHEMA"}}},"4":{"slot":4,"values":{"1":{"slot":1,"type":"STRING","value":"main.TABLE_OWNER"},"2":{"slot":2,"type":"STRING","value":"TABLE_OWNER"}}},"5":{"slot":5,"values":{"1":{"slot":1,"type":"STRING","value":"main.TABLE_TYPE"},"2":{"slot":2,"type":"STRING","value":"TABLE_TYPE"}}},"6":{"slot":6,"values":{"1":{"slot":1,"type":"STRING","value":"main.IS_TRANSIENT"},"2":{"slot":2,"type":"STRING","value":"IS_TRANSIENT"}}},"7":{"slot":7,"values":{"1":{"slot":1,"type":"STRING","value":"main.CLUSTERING_KEY"},"2":{"slot":2,"type":"STRING","value":"CLUSTERING_KEY"}}},"8":{"slot":8,"values":{"1":{"slot":1,"type":"STRING","value":"main.ROW_COUNT"},"2":{"slot":2,"type":"STRING","value":"ROW_COUNT"}}},"9":{"slot":9,"values":{"1":{"slot":1,"type":"STRING","value":"main.BYTES"},"2":{"slot":2,"type":"STRING","value":"BYTES"}}},"10":{"slot":10,"values":{"1":{"slot":1,"type":"STRING","value":"main.SUM_BYTES_IN_MB"},"2":{"slot":2,"type":"STRING","value":"SUM_BYTES_IN_MB"}}},"11":{"slot":11,"values":{"1":{"slot":1,"type":"STRING","value":"main.RETENTION_TIME"},"2":{"slot":2,"type":"STRING","value":"RETENTION_TIME"}}},"12":{"slot":12,"values":{"1":{"slot":1,"type":"STRING","value":"main.CREATED"},"2":{"slot":2,"type":"STRING","value":"CREATED"}}},"13":{"slot":13,"values":{"1":{"slot":1,"type":"STRING","value":"main.LAST_ALTERED"},"2":{"slot":2,"type":"STRING","value":"LAST_ALTERED"}}},"14":{"slot":14,"values":{"1":{"slot":1,"type":"STRING","value":"main.LAST_ALTERED_RANK"},"2":{"slot":2,"type":"STRING","value":"LAST_ALTERED_RANK"}}},"15":{"slot":15,"values":{"1":{"slot":1,"type":"STRING","value":"main.AUTO_CLUSTERING_ON"},"2":{"slot":2,"type":"STRING","value":"AUTO_CLUSTERING_ON"}}},"16":{"slot":16,"values":{"1":{"slot":1,"type":"STRING","value":"main.COMMENT"},"2":{"slot":2,"type":"STRING","value":"COMMENT"}}},"17":{"slot":17,"values":{"1":{"slot":1,"type":"STRING","value":"join.SCHEMA_TOTAL_BYTES_IN_MB"},"2":{"slot":2,"type":"STRING","value":"SCHEMA_TOTAL_BYTES_IN_MB"}}},"18":{"slot":18,"values":{"1":{"slot":1,"type":"STRING","value":"join.SCHEMA_LATEST_ALTERED_TABLE"},"2":{"slot":2,"type":"STRING","value":"SCHEMA_LATEST_ALTERED_TABLE"}}},"19":{"slot":19,"values":{"1":{"slot":1,"type":"STRING","value":"join.SCHEMA_OLDEST_ALTERED_TABLE"},"2":{"slot":2,"type":"STRING","value":"SCHEMA_OLDEST_ALTERED_TABLE"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"10325":{"id":10325,"inputCardinality":"ONE","outputCardinality":"MANY","implementationID":1716658327,"x":988,"y":358,"width":32,"height":32,"inputConnectorIDs":[10329],"outputConnectorIDs":[10327],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Add audit timestamp"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Include Input Columns","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Yes"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"Calculations","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"CURRENT_TIMESTAMP(6)"},"2":{"slot":2,"type":"STRING","value":"PROCESS_CREATE_TIMESTAMP"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"},"10326":{"id":10326,"inputCardinality":"ONE","outputCardinality":"ZERO","implementationID":-319648201,"x":1146,"y":355,"width":32,"height":32,"inputConnectorIDs":[10327],"outputConnectorIDs":[],"parameters":{"1":{"slot":1,"name":"Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Create Profiled Schema View"}}}},"visible":true,"status":{},"columnNames":[]},"2":{"slot":2,"name":"Schema","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${jv_schema_name}"}}}},"visible":true,"status":{},"columnNames":[]},"3":{"slot":3,"name":"View Name","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"VW_${jv_schema_name}_PROFILED"}}}},"visible":true,"status":{},"columnNames":[]},"5":{"slot":5,"name":"Database","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"${jv_database_name}"}}}},"visible":true,"status":{},"columnNames":[]},"6":{"slot":6,"name":"Secure View","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"No"}}}},"visible":true,"status":{},"columnNames":[]},"7":{"slot":7,"name":"View Type","elements":{"1":{"slot":1,"values":{"1":{"slot":1,"type":"STRING","value":"Standard"}}}},"visible":true,"status":{},"columnNames":[]}},"exportMappings":{},"compStatus":{},"validationStatus":{},"requiresViewRecreation":false,"expectedFailure":null,"activationStatus":"ENABLED"}},"connectors":{"10327":{"id":10327,"sourceID":10325,"targetID":10326},"10328":{"id":10328,"sourceID":10323,"targetID":10324},"10329":{"id":10329,"sourceID":10324,"targetID":10325},"10330":{"id":10330,"sourceID":10322,"targetID":10324}},"canUndo":true,"undoCommand":"Set Job Variables","undoCreated":1700579568010,"canRedo":false,"redoCommand":"","redoCreated":-1,"notes":{"10331":{"id":10331,"x":523,"y":167,"width":747,"height":357,"text":"Queries are run against INFORMATION_SCHEMA.TABLES for the selected database. Table Profile metadata is the main flow, which is enriched with Schema Profile data to create a flattened dataset to query the table metadata stored in Snowflake. ","colour":"00ce4f"}},"noteConnectors":{},"variables":{"jv_schema_name":{"definition":{"name":"jv_schema_name","type":"TEXT","scope":"TASKBATCH","description":"Name of the schema to profile","visibility":"PUBLIC"},"value":"${environment_default_schema}"},"jv_database_name":{"definition":{"name":"jv_database_name","type":"TEXT","scope":"TASKBATCH","description":"Name of the database containing the schema to profile","visibility":"PUBLIC"},"value":"${environment_database}"}},"grids":{}}}}],"version":"1.68.3","environment":"snowflake"}