Avoids exception when pandas.io.json.json_normalize contains items in meta parameter that don't always occur in every item of the list

Added documentation and test for issue #14505
Added keyword errors {'raise'|'ignore}
Shortened what's new
Removed commas in dictionary for linting compatibility
Updated doc

Author: dickreuter

pandas/io/json.py

@@ -723,7 +723,9 @@ def nested_to_record(ds, prefix="", level=0):
 
 def json_normalize(data, record_path=None, meta=None,
                    meta_prefix=None,
-                   record_prefix=None):
+                   record_prefix=None,
+                   errors='raise'):
+
     """
     "Normalize" semi-structured JSON data into a flat table
 
@@ -740,6 +742,8 @@ def json_normalize(data, record_path=None, meta=None,
         If True, prefix records with dotted (?) path, e.g. foo.bar.field if
         path to records is ['foo', 'bar']
     meta_prefix : string, default None
+    error: {'raise', 'ignore'}, default 'raise'
+        * ignore: will ignore keyErrors if keys listed in meta are not always present
 
     Returns
     -------
@@ -839,7 +843,14 @@ def _recursive_extract(data, path, seen_meta, level=0):
                     if level + 1 > len(val):
                         meta_val = seen_meta[key]
                     else:
-                        meta_val = _pull_field(obj, val[level:])
+                        try:
+                            meta_val = _pull_field(obj, val[level:])
+                        except KeyError as e:
+                            if errors == 'ignore':
+                                meta_val = np.nan
+                            else:
+                                raise KeyError(
+                                    "Try running with errors='ignore' as the following key may not always be present: " + str(e))
                     meta_vals[key].append(meta_val)
 
                 records.extend(recs)

pandas/io/tests/json/test_json_norm.py

@@ -225,6 +225,59 @@ def test_nested_flattens(self):
 
         self.assertEqual(result, expected)
 
+
+    def test_json_normalise_fix(self):
+        # issue 14505
+        j = {
+            "Trades": [{
+                "general": {
+                    "tradeid": 100,
+                    "trade_version": 1,
+                    "stocks": [{
+
+                        "symbol": "AAPL",
+                        "name": "Apple",
+                        "price": "0"
+
+                    }, {
+
+                        "symbol": "GOOG",
+                        "name": "Google",
+                        "price": "0"
+
+                    }
+                    ]
+                }
+            }, {
+                "general": {
+                    "tradeid": 100,
+                    "stocks": [{
+
+                        "symbol": "AAPL",
+                        "name": "Apple",
+                        "price": "0"
+
+                    }, {
+                        "symbol": "GOOG",
+                        "name": "Google",
+                        "price": "0"
+
+                    }
+                    ]
+                }
+            }
+            ]
+        }
+        j = json_normalize(data=j['Trades'], record_path=[['general', 'stocks']],
+                           meta=[['general', 'tradeid'], ['general', 'trade_version']], errors='ignore')
+        expected={'general.trade_version': {0: 1.0, 1: 1.0, 2: '', 3: ''},
+         'general.tradeid': {0: 100, 1: 100, 2: 100, 3: 100},
+         'name': {0: 'Apple', 1: 'Google', 2: 'Apple', 3: 'Google'},
+         'price': {0: '0', 1: '0', 2: '0', 3: '0'},
+         'symbol': {0: 'AAPL', 1: 'GOOG', 2: 'AAPL', 3: 'GOOG'}}
+
+        self.assertEqual(j.fillna('').to_dict(), expected)
+
 if __name__ == '__main__':
     nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb',
                          '--pdb-failure', '-s'], exit=False)