news-ingestion-dataset.json

{
  "entities": [
    {
      "typeName": "DataFile",
      "createdBy": "ingestors_news",
      "attributes": {
        "qualifiedName": "news_from_reuters",
        "uri": "news_from_reuters",
        "name": "news_from_reuters",

        "file_name_pattern": "news_from_reuters_DDMMYYY",
        "directory": "/landingzone/reuters/news",
        "frequency":"daily",
        "user":"reuters_webscrapper",
        "group":"newssource",
        "format":"csv",
        "server" : {"guid": -100,"typeName": "server"},
        "schema":[
          { "col" : "id" ,"data_type" : "string" ,"required" : true },
          { "col" : "scrap_time" ,"data_type" : "timestamp" ,"required" : true },
          { "col" : "url" ,"data_type" : "string" ,"required" : true },
          { "col" : "headline" ,"data_type" : "string" ,"required" : true },
          { "col" : "content" ,"data_type" : "string" ,"required" : false }
        ]
      },
      "classifications": [
        { "typeName": "news" }
      ]
    },

    { "typeName": "kafka_value_message_schema","guid":-500,
      "createdBy": "ingestors_news",
      "attributes": {
        "qualifiedName": "news_topic.id",
        "uri": "news_topic.id",
        "name": "id",
        "col": "id",
        "data_type": "string",
        "required": true
      }
    },
    { "typeName": "kafka_value_message_schema","guid":-501,
      "createdBy": "ingestors_news",
      "attributes": {
        "qualifiedName": "news_topic.scrap_time",
        "uri": "news_topic.scrap_time",
        "name": "scrap_time",
        "col": "scrap_time",
        "data_type": "timestamp",
        "required": true
      }
    },
    { "typeName": "kafka_value_message_schema","guid":-502,
      "createdBy": "ingestors_news",
      "attributes": {
        "qualifiedName": "news_topic.url",
        "uri": "news_topic.url",
        "name": "url",
        "col": "url",
        "data_type": "string",
        "required": true
      }
    },
    { "typeName": "kafka_value_message_schema","guid":-503,
      "createdBy": "ingestors_news",
      "attributes": {
        "qualifiedName": "news_topic.headline",
        "uri": "news_topic.headline",
        "name": "headline",
        "col": "headline",
        "data_type": "string",
        "required": true
      }
    },
    { "typeName": "kafka_value_message_schema","guid":-504,
      "createdBy": "ingestors_news",
      "attributes": {
        "qualifiedName": "news_topic.content",
        "uri": "news_topic.content",
        "name": "content",
        "col": "content",
        "data_type": "string",
        "required": false
      }
    },
    {
      "typeName": "kafka_topic_and_schema",
      "createdBy": "ingestors_news",
      "attributes": {
        "qualifiedName": "news_topic",
        "uri": "news_topic",
        "name": "news_topic",
        "topic":"news_topic",
        "description": "Topics on to news from different websites are injected",
        "owner": "ingestors_news",
        "key_schema":[
          { "typeName": "kafka_value_message_schema","guid":-500 }
        ],
        "value_schema":[
          { "typeName": "kafka_value_message_schema","guid":-500 },
          { "typeName": "kafka_value_message_schema","guid":-501 },
          { "typeName": "kafka_value_message_schema","guid":-502 },
          { "typeName": "kafka_value_message_schema","guid":-503 },
          { "typeName": "kafka_value_message_schema","guid":-504 }
        ]
      },
      "classifications": [
        { "typeName": "news" }
      ]
    },

    {
      "typeName": "hbase_table",
      "createdBy": "ingestors",
      "guid":-110 ,
      "attributes": {
        "qualifiedName": "news:news_from_reuters",
        "uri": "news:news_from_reuters",
        "name": "news_from_reuters",
        "description": "Table which contains news scrapped from reuters",
        "owner": "ingestors"
      }
    },
      {
      "typeName": "hbase_column_family",
      "createdBy": "ingestors",
      "guid":-111,
      "attributes": {
        "qualifiedName": "news:news_from_reuters.z",
        "uri": "news:news_from_reuters.z",
        "name": "z",
        "table":{ "qualifiedName":"news:news_from_reuters" ,"guid":-110 ,"typeName": "hbase_table"}
      }
    }
        ,{
      "typeName": "hbase_column",
      "createdBy": "ingestors",
      "attributes": {
        "qualifiedName": "news:news_from_reuters.z.message",
        "uri": "news:news_from_reuters.z.message",
        "name": "message",
        "type": "string",
        "column_family":{ "qualifiedName":"news:news_from_reuters.z" ,"guid":-111 ,"typeName": "hbase_column_family"}
      }
    }
  ],
  "referredEntities": {
    "-100": {
      "guid": "-100",
      "typeName": "server",
      "attributes": {
        "qualifiedName": "landing_zone_server_1@dev",
        "uri": "landing_zone_server_1@dev",
        "name": "landing_zone_server_1",
        "dns_name": "pluto",
        "ip_address": "10.71.68.93"
      }
    }
  }
}