Airtable schema

packages/external-db-airtable/lib/airtable_schema_provider.js

@@ -1,69 +1,47 @@
-// const { promisify } = require('util')
-// const { translateErrorCodes } = require('./sql_exception_translator')
-// const SchemaColumnTranslator = require('./sql_schema_translator')
-// const { escapeId, escapeTable } = require('./mysql_utils')
-// const { SystemFields, validateSystemFields, asWixSchema, parseTableData } = require('velo-external-db-commons')
-// const Airtable = require ('airtable')
+const SchemaColumnTranslator = require('./sql_schema_translator')
+const { asWixSchema } = require('velo-external-db-commons')
 
+const axios = require('axios').create({
+    baseURL: 'https://api.airtable.com',
+});
 
 
-// class SchemaProvider {
-//     constructor(pool) {
-//         this.pool = pool
+class SchemaProvider {
+    constructor(pool, apiKey, metaApiKey, test) {
+        this.pool = pool
+        this.apiKey = apiKey
+        this.metaApiKey = metaApiKey
+        this.axios = axios
+        this.sqlSchemaTranslator = new SchemaColumnTranslator()
+    }
 
-//         this.sqlSchemaTranslator = new SchemaColumnTranslator()
+    async list() {
+        const baseId = this.pool.getId()
 
-//         this.query = promisify(this.pool.query).bind(this.pool)
-//     }
+        const response = await this.axios.get(`v0/meta/bases/${baseId}/tables`, {
+            headers: {
+                'Authorization': `Bearer ${this.apiKey}`,
+                'X-Airtable-Client-Secret': this.metaApiKey
+            }
+        })
+        const tables = response.data.tables.reduce((pV, cV) => ({
+            ...pV,
+            [cV.name]:
+                { fields: cV.fields.map(field => { return { field: field.name, type: this.translateDbTypes(field.type) } }) }
+        }), {})
 
-//     async list() {
-//         const currentDb = this.pool.config.connectionConfig.database
-//         const data = await this.query('SELECT TABLE_NAME as table_name, COLUMN_NAME as field, DATA_TYPE as type FROM information_schema.columns WHERE TABLE_SCHEMA = ? ORDER BY TABLE_NAME, ORDINAL_POSITION', currentDb)
-//         const tables = parseTableData( data )
-//         return Object.entries(tables)
-//                      .map(([collectionName, rs]) => asWixSchema(rs.map( this.translateDbTypes.bind(this) ), collectionName))
-//     }
+        return Object.entries(tables).map(([collectionName, rs]) => asWixSchema(rs.fields, collectionName))
+    }
 
 
-//     async create(collectionName, columns) {
-//         const dbColumnsSql = [...SystemFields, ...(columns || [])].map( c => this.sqlSchemaTranslator.columnToDbColumnSql(c) )
-//                                                                        .join(', ')
-//         const primaryKeySql = SystemFields.filter(f => f.isPrimary).map(f => escapeId(f.name)).join(', ')
+    async describeCollection(collectionName) {
+        return (await this.list()).filter(schemas => schemas.id === collectionName)
+    }
 
-//         await this.query(`CREATE TABLE IF NOT EXISTS ${escapeTable(collectionName)} (${dbColumnsSql}, PRIMARY KEY (${primaryKeySql}))`,
-//                          [...(columns || []).map(c => c.name)])
-//                   .catch( translateErrorCodes )
-//     }
+    translateDbTypes(type) {
+        return this.sqlSchemaTranslator.translateType(type)
+    }
 
-//     async drop(collectionName) {
-//         await this.query(`DROP TABLE IF EXISTS ${escapeTable(collectionName)}`)
-//                   .catch( translateErrorCodes )
-//     }
+}
 
-//     async addColumn(collectionName, column) {
-//         await validateSystemFields(column.name)
-//         await this.query(`ALTER TABLE ${escapeTable(collectionName)} ADD ${escapeId(column.name)} ${this.sqlSchemaTranslator.dbTypeFor(column)}`)
-//                   .catch( translateErrorCodes )
-//     }
-
-//     async removeColumn(collectionName, columnName) {
-//         await validateSystemFields(columnName)
-//         return await this.query(`ALTER TABLE ${escapeTable(collectionName)} DROP COLUMN ${escapeId(columnName)}`)
-//                          .catch( translateErrorCodes )
-//     }
-
-//     async describeCollection(collectionName) {
-//         const res = await this.query(`DESCRIBE ${escapeTable(collectionName)}`)
-//                               .catch( translateErrorCodes )
-//         const rows = res.map(r => ({field: r.Field, type: r.Type}))
-//                         .map( this.translateDbTypes.bind(this) )
-//         return asWixSchema(rows, collectionName)
-//     }
-
-//     translateDbTypes(row) {
-//         row.type = this.sqlSchemaTranslator.translateType(row.type)
-//         return row
-//     }
-// }
-
-// module.exports = SchemaProvider
+module.exports = SchemaProvider

packages/external-db-airtable/lib/connection_provider.js

@@ -1,14 +1,18 @@
 const DataProvider = require('./airtable_data_provider')
+const SchemaProvider = require ('./airtable_schema_provider')
 const FilterParser = require('./sql_filter_transformer')
 const DatabaseOperations = require('./airtable_operations')
 const Airtable = require('airtable')
 const init = async (cfg, _poolOptions) => {
     const options = _poolOptions || {}
-    const pool = new Airtable({ apiKey: cfg.privateApiKey,...options }).base(cfg.baseId)
+    const pool = new Airtable({ apiKey: cfg.apiPrivateKey,...options }).base(cfg.baseId)
     const databaseOperations = new DatabaseOperations(pool)
     const filterParser = new FilterParser()
     const dataProvider = new DataProvider(pool, filterParser)
-    return { dataProvider: dataProvider, connection: pool, databaseOperations, cleanup: async () => { } }
+    const schemaProvider = new SchemaProvider(pool, cfg.apiPrivateKey, cfg.metaApiKey)
+    if (process.env.NODE_ENV == 'test') 
+        schemaProvider.axios.defaults.baseURL = `http://localhost:9000/`
+    return { dataProvider, schemaProvider, connection: pool, databaseOperations, cleanup: async () => { } }
 
 }
 

packages/external-db-airtable/lib/sql_schema_translator.js

@@ -0,0 +1,51 @@
+
+class SchemaColumnTranslator {
+
+    constructor() {
+    }
+
+    translateType(dbType) {
+        const type = dbType.toLowerCase()
+            .split('(')
+            .shift()
+
+        switch (type) {
+            case 'autonumber':
+            case 'count':
+            case 'currency':
+            case 'duration':
+            case 'number':
+            case 'rating':
+            case 'percent':
+                return 'number'
+
+            case 'date':
+            case 'createdtime':
+            case 'datetime':
+                return 'datetime'
+
+            case 'email':
+            case 'longtext':
+            case 'phonenumber':
+            case 'rollup':
+            case 'singlelinetext':
+            case 'singleselect':
+            case 'url':
+            case 'multilinetext':
+            case 'linkedrecord':
+            case 'multiplerecordlinks':
+            case 'multipleattachment':
+                return 'text'
+
+            case 'checkbox':
+                return 'boolean'
+
+            default:
+                console.log(type)
+                throw Error(type)
+        }
+    }
+
+}
+
+module.exports = SchemaColumnTranslator

packages/external-db-airtable/lib/sql_schema_translator.spec.js

@@ -0,0 +1,65 @@
+const SchemaColumnTranslator = require('./sql_schema_translator')
+const { Uninitialized } = require('test-commons')
+const Chance = require('chance')
+const chance = Chance();
+
+describe('Sql Schema Column Translator', () => {
+
+    describe('translate db type to velo type', () => {
+        describe('numeric fields', () => {
+
+            test('integer', () => {
+                ['Autonumber', 'Count', 'Duration', 'Number','Rating','Percent'].forEach(t => {
+                    expect( env.schemaTranslator.translateType(t) ).toEqual('number')
+                })
+            })
+
+            test('decimal float', () => {
+                ['Currency'].forEach(t => {
+                    expect( env.schemaTranslator.translateType(t) ).toEqual('number')
+                })
+            })
+        })
+
+        describe('string fields', () => {
+            test('string', () => {
+                ['Email', 'Longtext', 'Phonenumber', 'Rollup', 'Singlelinetext','Singleselect','URL',
+                 'multilinetext','linkedrecord','multiplerecordlinks','multipleattachment'].forEach(t => {
+                    expect( env.schemaTranslator.translateType(t) ).toEqual('text')
+                })
+            })
+        })
+
+        describe('date time fields', () => {
+            test('date', () => {
+                ['Createdtime','Date','datetime'].forEach(t => {
+                    expect( env.schemaTranslator.translateType(t) ).toEqual('datetime')
+                })
+            })
+        })
+
+        describe('other fields', () => {
+            test('boolean', () => {
+                ['Checkbox'].forEach(t => {
+                    expect( env.schemaTranslator.translateType(t) ).toEqual('boolean')
+                })
+            })
+        })
+    })
+
+    const ctx = {
+        fieldName: Uninitialized,
+    };
+
+    const env = {
+        schemaTranslator: Uninitialized,
+    };
+
+    beforeEach(() => {
+        ctx.fieldName = chance.word();
+    });
+
+    beforeAll(function() {
+        env.schemaTranslator = new SchemaColumnTranslator()
+    });
+})

packages/external-db-airtable/package.json

@@ -27,6 +27,7 @@
   },
   "dependencies": {
     "airtable": "^0.11.1",
+    "axios": "^0.23.0",
     "express": "^4.17.1",
     "velo-external-db-commons": "^0.0.0"
   },

packages/external-db-airtable/tests/drivers/db_operations_test_support.js

@@ -3,19 +3,20 @@ const init = require('../../lib/connection_provider')
 
 const createPool = modify => {
     const config = {
-        privateApiKey: 'key123',
+        apiPrivateKey: 'key123',
+        metaApiKey: 'meta123',
         baseId: 'app123'
     }
 
     const { connection, cleanup } =  init({ ...config, ...modify }, { endpointUrl: 'http://localhost:9000', requestTimeout: 1000 })
     return { connection, cleanup }
 }
 
-const dbOperationWithMisconfiguredPassword = () => new DatabaseOperations(createPool({ privateApiKey: 'wrong' }).connection)
+const dbOperationWithMisconfiguredPassword = () => new DatabaseOperations(createPool({ apiPrivateKey: 'wrong' }).connection)
 
 const dbOperationWithMisconfiguredDatabase = () => new DatabaseOperations(createPool({ baseId: 'wrong' }).connection)
 
-const dbOperationWithMisconfiguredHost = () => new DatabaseOperations(createPool({ privateApiKey: 'wrong' }).connection)
+const dbOperationWithMisconfiguredHost = () => new DatabaseOperations(createPool({ apiPrivateKey: 'wrong' }).connection)
 
 const dbOperationWithValidDB = () => {
     const { connection, cleanup } = createPool({ } )

packages/external-db-airtable/tests/drivers/mock_air_table.js

@@ -20,10 +20,17 @@ const _checkParamsMiddleware = (req, res, next) => {
     if (req.params.baseId !== 'app123')
         return next(new AirtableError('NOT_FOUND', 'Could not find what you are looking for', '404'))
 
-    if (req.params.tableIdOrName !== 'Table')
+    if (req.params.tableIdOrName && req.params.tableIdOrName !== 'Table')
         return next(new AirtableError('NOT_FOUND', `Could not find table ${req.params.tableIdOrName} in application ${req.params.baseId}`, '404'))
+
     next()
 }
+const _checkParamsMetaMiddleware = ( _checkParamsMiddleware, function (req,res,next) { //TODO: refactor middleware
+    console.log(req.get('X-Airtable-Client-Secret'));
+    if (req.get('X-Airtable-Client-Secret') !== 'meta123')
+        return next(new AirtableError('Unauthorized','Unauthorized','401'))
+    next()
+})
 
 
 const app = express()
@@ -118,6 +125,48 @@ app.delete('/v0/:baseId/:tableIdOrName', _checkParamsMiddleware, function (req,
     res.json({ records })
 });
 
+
+// *************** Meta API **********************
+
+app.get('/v0/meta/bases/:baseId/tables',_checkParamsMetaMiddleware, function (req, res) {
+    res.json({
+        "tables": [
+          {
+            "id": "tbltp8DGLhqbUmjK1",
+            "name": "Table",
+            "primaryFieldId": "fld1VnoyuotSTyxW1",
+            "fields": [
+              {
+                "id": "fld1VnoyuotSTyxW1",
+                "name": "Name",
+                "type": "singleLineText"
+              },
+              {
+                "id": "fldoaIqdn5szURHpw",
+                "name": "Age",
+                "type": "number"
+              },
+              {
+                "id": "fldumZe00w09RYTW6",
+                "name": "URL",
+                "type": "url",
+              }
+            ],
+            "views": [
+              {
+                "id": "viwQpsuEDqHFqegkp",
+                "name": "Grid view",
+                "type": "grid"
+              }
+            ]
+          }
+        ]
+      }
+    )
+})
+
+
+
 app.use(function (req, res) {
     res.status(404);
     res.json({ error: 'NOT_FOUND' });

packages/external-db-config/lib/readers/gcp_config_reader.js

@@ -83,13 +83,13 @@ class GcpAirtableConfigReader {
   constructor() { }
 
   async readConfig() {
-    const { AIRTABLE_API_KEY, BASE_ID, SECRET_KEY } = process.env
-    return { apiPrivateKey: AIRTABLE_API_KEY, baseId: BASE_ID, secretKey: SECRET_KEY }
+    const { AIRTABLE_API_KEY, META_API_KEY, BASE_ID, SECRET_KEY } = process.env
+    return { apiPrivateKey: AIRTABLE_API_KEY,metaApiKey: META_API_KEY, baseId: BASE_ID, secretKey: SECRET_KEY }
   }
 
   validate() {
     return {
-      missingRequiredSecretsKeys: checkRequiredKeys(process.env, ['AIRTABLE_API_KEY', 'BASE_ID', 'SECRET_KEY'])
+      missingRequiredSecretsKeys: checkRequiredKeys(process.env, ['AIRTABLE_API_KEY', 'META_API_KEY', 'BASE_ID', 'SECRET_KEY'])
     }
   }
 }

packages/velo-external-db/test/resources/engines/airtable_resources.js

@@ -6,7 +6,7 @@ let _server
 const PORT = 9000
 
 const connection = async () => {
-    const { connection, cleanup } = await init({ privateApiKey: process.env.AIRTABLE_API_KEY, baseId: process.env.BASE_ID },
+    const { connection, cleanup } = await init({ apiPrivateKey: process.env.AIRTABLE_API_KEY, baseId: process.env.BASE_ID },
                                                 {endpointUrl: 'http://localhost:'+PORT, requestTimeout: 1000})
 
     return { pool: connection, cleanup: cleanup }
@@ -27,6 +27,7 @@ const shutdownEnv = async () => {
 
 const setActive = () => {
     process.env.AIRTABLE_API_KEY = 'key123'
+    process.env.META_API_KEY = 'meta123'
     process.env.TYPE = 'airtable'
     process.env.BASE_ID = 'app123'
 }