From 6744bd48b540aa1288acdc4bb67a3cd677ab17c9 Mon Sep 17 00:00:00 2001 From: Joseph Knox Date: Thu, 28 Jun 2018 09:54:53 -0700 Subject: [PATCH] Fixes #179 Looks to be a plug and play change between `pd.DataFrame.from_csv` and `pd.read_csv` since I don't see where anything more than the path argument has been supplied to the function call. --- allensdk/api/cache.py | 10 ++-- allensdk/core/mouse_connectivity_cache.py | 2 +- allensdk/test/api/test_cacheable.py | 58 +++++++++---------- allensdk/test/api/test_pager.py | 9 ++- .../test/core/test_cell_types_cache_unit.py | 47 ++++++++------- doc_template/data_api_client.rst | 2 +- 6 files changed, 63 insertions(+), 65 deletions(-) diff --git a/allensdk/api/cache.py b/allensdk/api/cache.py index 8d1f675e9..681acf800 100755 --- a/allensdk/api/cache.py +++ b/allensdk/api/cache.py @@ -233,7 +233,7 @@ def load_csv(self, index : string, optional post-rename column to use as the row label. ''' - data = pd.DataFrame.from_csv(path) + data = pd.read_csv(path) Cache.rename_columns(data, rename) @@ -372,14 +372,14 @@ def csv_writer(pth, gen): def cache_csv_json(): return { 'writer': Cache.csv_writer, - 'reader': lambda f: pd.DataFrame.from_csv(f).to_dict('records') + 'reader': lambda f: pd.read_csv(f).to_dict('records') } @staticmethod def cache_csv_dataframe(): return { 'writer': Cache.csv_writer, - 'reader' : pd.DataFrame.from_csv + 'reader' : pd.read_csv } @staticmethod @@ -411,7 +411,7 @@ def cache_json(): def cache_csv(): return { 'writer': Cache.csv_writer, - 'reader': pd.DataFrame.from_csv + 'reader': pd.read_csv } @staticmethod @@ -514,7 +514,7 @@ def wrap(self, fn, path, cache, else: data = ju.read(path) elif return_dataframe is True: - data = pd.DataFrame.from_csv(path) + data = pd.read_csv(path) else: raise ValueError( 'save_as_json=False cannot be used with return_dataframe=False') diff --git a/allensdk/core/mouse_connectivity_cache.py b/allensdk/core/mouse_connectivity_cache.py index 2d80dcc77..2bf982375 100644 --- a/allensdk/core/mouse_connectivity_cache.py +++ b/allensdk/core/mouse_connectivity_cache.py @@ -417,7 +417,7 @@ def get_experiment_structure_unionizes(self, experiment_id, pre=col_rn, post=filter_fn, writer=lambda p, x : pd.DataFrame(x).to_csv(p), - reader=pd.DataFrame.from_csv) + reader=pd.read_csv) def rank_structures(self, experiment_ids, is_injection, structure_ids=None, hemisphere_ids=None, rank_on='normalized_projection_volume', n=5, threshold=10**-2): diff --git a/allensdk/test/api/test_cacheable.py b/allensdk/test/api/test_cacheable.py index ec51f06dc..d43d3b55c 100644 --- a/allensdk/test/api/test_cacheable.py +++ b/allensdk/test/api/test_cacheable.py @@ -50,7 +50,7 @@ _msg = [{'whatever': True}] _pd_msg = pd.DataFrame(_msg) -_csv_msg = pd.DataFrame.from_csv(StringIO.StringIO(""",whatever +_csv_msg = pd.read_csv(StringIO.StringIO(""",whatever 0,True """)) @@ -59,8 +59,8 @@ @patch("allensdk.core.json_utilities.read", return_value=_msg) @patch("allensdk.core.json_utilities.read_url_get", return_value={'msg': _msg}) @patch('csv.DictWriter') -@patch.object(pd.DataFrame, 'from_csv', return_value=_csv_msg) -def test_cacheable_csv_dataframe(from_csv, dictwriter, ju_read_url_get, +@patch.object(pd, 'read_csv', return_value=_csv_msg) +def test_cacheable_csv_dataframe(read_csv, dictwriter, ju_read_url_get, ju_read, ju_write): @cacheable() def get_hemispheres(): @@ -79,7 +79,7 @@ def get_hemispheres(): ju_read_url_get.assert_called_once_with( 'http://api.brain-map.org/api/v2/data/query.json?q=model::Hemisphere') - from_csv.assert_called_once_with('/xyz/abc/example.txt') + read_csv.assert_called_once_with('/xyz/abc/example.txt') assert not ju_write.called, 'write should not have been called' assert not ju_read.called, 'read should not have been called' mkdir.assert_called_once_with('/xyz/abc') @@ -90,8 +90,8 @@ def get_hemispheres(): @patch("allensdk.core.json_utilities.read", return_value=_msg) @patch("allensdk.core.json_utilities.read_url_get", return_value={'msg': _msg}) @patch.object(Manifest, 'safe_mkdir') -@patch.object(pd.DataFrame, 'from_csv', return_value=_csv_msg) -def test_cacheable_json(from_csv, mkdir, ju_read_url_get, ju_read, ju_write): +@patch.object(pd, 'read_csv', return_value=_csv_msg) +def test_cacheable_json(read_csv, mkdir, ju_read_url_get, ju_read, ju_write): @cacheable() def get_hemispheres(): return RmaApi().model_query(model='Hemisphere') @@ -104,7 +104,7 @@ def get_hemispheres(): ju_read_url_get.assert_called_once_with( 'http://api.brain-map.org/api/v2/data/query.json?q=model::Hemisphere') - assert not from_csv.called, 'from_csv should not have been called' + assert not read_csv.called, 'read_csv should not have been called' ju_write.assert_called_once_with('/xyz/abc/example.json', _msg) ju_read.assert_called_once_with('/xyz/abc/example.json') @@ -136,8 +136,8 @@ def get_hemispheres_excpt(): @patch("allensdk.core.json_utilities.write") @patch("allensdk.core.json_utilities.read", return_value=_msg) @patch("allensdk.core.json_utilities.read_url_get", return_value={'msg': _msg}) -@patch.object(pd.DataFrame, 'from_csv', return_value=_csv_msg) -def test_cacheable_no_cache_csv(from_csv, ju_read_url_get, ju_read, ju_write): +@patch.object(pd, 'read_csv', return_value=_csv_msg) +def test_cacheable_no_cache_csv(read_csv, ju_read_url_get, ju_read, ju_write): @cacheable() def get_hemispheres(): return RmaApi().model_query(model='Hemisphere') @@ -149,19 +149,19 @@ def get_hemispheres(): assert df.loc[:, 'whatever'][0] assert not ju_read_url_get.called - from_csv.assert_called_once_with('/xyz/abc/example.csv') + read_csv.assert_called_once_with('/xyz/abc/example.csv') assert not ju_write.called, 'json write should not have been called' assert not ju_read.called, 'json read should not have been called' @patch("pandas.io.json.read_json", return_value=_pd_msg) -@patch.object(pd.DataFrame, "from_csv", return_value=_csv_msg) +@patch.object(pd, "read_csv", return_value=_csv_msg) @patch("allensdk.core.json_utilities.write") @patch("allensdk.core.json_utilities.read", return_value=_msg) @patch("allensdk.core.json_utilities.read_url_get", return_value={'msg': _msg}) @patch.object(Manifest, 'safe_mkdir') def test_cacheable_json_dataframe(mkdir, ju_read_url_get, ju_read, ju_write, - from_csv, mock_read_json): + read_csv, mock_read_json): @cacheable() def get_hemispheres(): return RmaApi().model_query(model='Hemisphere') @@ -174,7 +174,7 @@ def get_hemispheres(): ju_read_url_get.assert_called_once_with( 'http://api.brain-map.org/api/v2/data/query.json?q=model::Hemisphere') - assert not from_csv.called, 'from_csv should not have been called' + assert not read_csv.called, 'read_csv should not have been called' mock_read_json.assert_called_once_with('/xyz/abc/example.json', orient='records') ju_write.assert_called_once_with('/xyz/abc/example.json', _msg) @@ -183,14 +183,14 @@ def get_hemispheres(): @patch("pandas.io.json.read_json", return_value=_pd_msg) -@patch.object(pd.DataFrame, "from_csv", return_value=_csv_msg) +@patch.object(pd, "read_csv", return_value=_csv_msg) @patch("allensdk.core.json_utilities.write") @patch("allensdk.core.json_utilities.read", return_value=_msg) @patch("allensdk.core.json_utilities.read_url_get", return_value={'msg': _msg}) @patch('csv.DictWriter') @patch.object(Manifest, 'safe_mkdir') def test_cacheable_csv_json(mkdir, dictwriter, ju_read_url_get, ju_read, - ju_write, from_csv, mock_read_json): + ju_write, read_csv, mock_read_json): @cacheable() def get_hemispheres(): return RmaApi().model_query(model='Hemisphere') @@ -207,7 +207,7 @@ def get_hemispheres(): ju_read_url_get.assert_called_once_with( 'http://api.brain-map.org/api/v2/data/query.json?q=model::Hemisphere') - from_csv.assert_called_once_with('/xyz/example.csv') + read_csv.assert_called_once_with('/xyz/example.csv') dictwriter.return_value.writerow.assert_called() assert not mock_read_json.called, 'pj.read_json should not have been called' assert not ju_write.called, 'ju.write should not have been called' @@ -219,9 +219,9 @@ def get_hemispheres(): @patch("allensdk.core.json_utilities.write") @patch("allensdk.core.json_utilities.read", return_value=_msg) @patch("allensdk.core.json_utilities.read_url_get", return_value={'msg': _msg}) -@patch.object(pd.DataFrame, "from_csv") +@patch.object(pd, "read_csv") @patch.object(pd.DataFrame, "to_csv") -def test_cacheable_no_save(to_csv, from_csv, ju_read_url_get, ju_read, +def test_cacheable_no_save(to_csv, read_csv, ju_read_url_get, ju_read, ju_write): @cacheable() def get_hemispheres(): @@ -234,7 +234,7 @@ def get_hemispheres(): ju_read_url_get.assert_called_once_with( 'http://api.brain-map.org/api/v2/data/query.json?q=model::Hemisphere') assert not to_csv.called, 'to_csv should not have been called' - assert not from_csv.called, 'from_csv should not have been called' + assert not read_csv.called, 'read_csv should not have been called' assert not ju_write.called, 'json write should not have been called' assert not ju_read.called, 'json read should not have been called' @@ -242,9 +242,9 @@ def get_hemispheres(): @patch("allensdk.core.json_utilities.write") @patch("allensdk.core.json_utilities.read", return_value=_msg) @patch("allensdk.core.json_utilities.read_url_get", return_value={'msg': _msg}) -@patch.object(pd.DataFrame, "from_csv", return_value=_csv_msg) +@patch.object(pd, "read_csv", return_value=_csv_msg) @patch.object(pd.DataFrame, "to_csv") -def test_cacheable_no_save_dataframe(to_csv, from_csv, ju_read_url_get, +def test_cacheable_no_save_dataframe(to_csv, read_csv, ju_read_url_get, ju_read, ju_write): @cacheable() def get_hemispheres(): @@ -257,19 +257,19 @@ def get_hemispheres(): ju_read_url_get.assert_called_once_with( 'http://api.brain-map.org/api/v2/data/query.json?q=model::Hemisphere') assert not to_csv.called, 'to_csv should not have been called' - assert not from_csv.called, 'from_csv should not have been called' + assert not read_csv.called, 'read_csv should not have been called' assert not ju_write.called, 'json write should not have been called' assert not ju_read.called, 'json read should not have been called' -@patch.object(pd.DataFrame, "from_csv", return_value=_csv_msg) +@patch.object(pd, "read_csv", return_value=_csv_msg) @patch("allensdk.core.json_utilities.write") @patch("allensdk.core.json_utilities.read", return_value=_msg) @patch("allensdk.core.json_utilities.read_url_get", return_value={'msg': _msg}) @patch('csv.DictWriter') @patch.object(Manifest, 'safe_mkdir') def test_cacheable_lazy_csv_no_file(mkdir, dictwriter, ju_read_url_get, - ju_read, ju_write, from_csv): + ju_read, ju_write, read_csv): @cacheable() def get_hemispheres(): return RmaApi().model_query(model='Hemisphere') @@ -289,7 +289,7 @@ def get_hemispheres(): 'http://api.brain-map.org/api/v2/data/query.json?q=model::Hemisphere') open_mock.assert_called_once_with('/xyz/abc/example.csv', 'w') dictwriter.return_value.writerow.assert_called() - from_csv.assert_called_once_with('/xyz/abc/example.csv') + read_csv.assert_called_once_with('/xyz/abc/example.csv') assert not ju_write.called, 'json write should not have been called' assert not ju_read.called, 'json read should not have been called' @@ -297,8 +297,8 @@ def get_hemispheres(): @patch("allensdk.core.json_utilities.write") @patch("allensdk.core.json_utilities.read", return_value=_msg) @patch("allensdk.core.json_utilities.read_url_get", return_value={'msg': _msg}) -@patch.object(pd.DataFrame, "from_csv", return_value=_csv_msg) -def test_cacheable_lazy_csv_file_exists(from_csv, ju_read_url_get, ju_read, +@patch.object(pd, "read_csv", return_value=_csv_msg) +def test_cacheable_lazy_csv_file_exists(read_csv, ju_read_url_get, ju_read, ju_write): @cacheable() def get_hemispheres(): @@ -312,6 +312,6 @@ def get_hemispheres(): assert df.loc[:, 'whatever'][0] assert not ju_read_url_get.called - from_csv.assert_called_once_with('/xyz/abc/example.csv') + read_csv.assert_called_once_with('/xyz/abc/example.csv') assert not ju_write.called, 'json write should not have been called' - assert not ju_read.called, 'json read should not have been called' \ No newline at end of file + assert not ju_read.called, 'json read should not have been called' diff --git a/allensdk/test/api/test_pager.py b/allensdk/test/api/test_pager.py index 3827fada9..9b2afae0d 100644 --- a/allensdk/test/api/test_pager.py +++ b/allensdk/test/api/test_pager.py @@ -38,7 +38,6 @@ from allensdk.api.queries.rma_pager import RmaPager, pageable from allensdk.api.queries.rma_api import RmaApi import allensdk.core.json_utilities as ju -import pandas.io.json as pj import pandas as pd from six.moves import builtins import os @@ -58,7 +57,7 @@ def pager(): _msg = [{'whatever': True}] _pd_msg = pd.DataFrame(_msg) -_csv_msg = pd.DataFrame.from_csv(StringIO.StringIO(""",whatever +_csv_msg = pd.read_csv(StringIO.StringIO(""",whatever 0,True """)) @@ -145,11 +144,11 @@ def get_genes(**kwargs): (Cache.cache_csv, Cache.cache_csv_json, Cache.cache_csv_dataframe)) -@patch.object(pd.DataFrame, "from_csv", return_value=_csv_msg) +@patch.object(pd, "read_csv", return_value=_csv_msg) @patch("allensdk.core.json_utilities.read_url_get", side_effect=_read_url_get_msg5) @patch("os.makedirs") -def test_cacheable_pageable_csv(os_makedirs, ju_read_url_get, from_csv, +def test_cacheable_pageable_csv(os_makedirs, ju_read_url_get, read_csv, cache_style): archive_templates = \ {"cam_cell_queries": [ @@ -193,7 +192,7 @@ def get_cam_cell_metrics(*args, [0, 1, 2, 3, 4, 5]) assert ju_read_url_get.call_args_list == list(expected_calls) - from_csv.assert_called_once_with('/path/to/cam_cell_metrics.csv') + read_csv.assert_called_once_with('/path/to/cam_cell_metrics.csv') assert csv_writerow.call_args_list == [call({'whatever': 'whatever'}), call({'whatever': True}), diff --git a/allensdk/test/core/test_cell_types_cache_unit.py b/allensdk/test/core/test_cell_types_cache_unit.py index 0d8c096e9..22e6db21a 100644 --- a/allensdk/test/core/test_cell_types_cache_unit.py +++ b/allensdk/test/core/test_cell_types_cache_unit.py @@ -36,13 +36,12 @@ import allensdk.core.cell_types_cache as CTC from allensdk.core.cell_types_cache import ReporterStatus as RS import pytest -from pandas.core.frame import DataFrame +import pandas as pd from allensdk.config import enable_console_log from mock import MagicMock, patch, call, mock_open from six.moves import builtins import itertools as it import allensdk.core.json_utilities as ju -import pandas.io.json as pj import os _MOCK_PATH = '/path/to/xyz.txt' @@ -250,9 +249,9 @@ def test_get_reconstruction(cache_fixture, @pytest.mark.parametrize('path_exists', (False, True)) -@patch.object(DataFrame, "to_csv") -@patch.object(DataFrame, "from_csv") -def test_get_reconstruction_with_api(from_csv, +@patch.object(pd.DataFrame, "to_csv") +@patch.object(pd, "read_csv") +def test_get_reconstruction_with_api(read_csv, to_csv, cache_fixture, cell_id, @@ -279,9 +278,9 @@ def test_get_reconstruction_with_api(from_csv, assert query_mock.called -@patch.object(DataFrame, "to_csv") -@patch.object(DataFrame, "from_csv") -def test_get_reconstruction_exception(from_csv, +@patch.object(pd.DataFrame, "to_csv") +@patch.object(pd, "read_csv") +def test_get_reconstruction_exception(read_csv, to_csv, cache_fixture, cell_id): @@ -410,9 +409,9 @@ def test_get_ephys_features(cache_fixture, @pytest.mark.parametrize('df,path_exists', it.product((False,True), (False,True))) -@patch.object(DataFrame, "to_csv") -@patch.object(DataFrame, "from_csv") -def test_get_ephys_features_with_api(from_csv, +@patch.object(pd.DataFrame, "to_csv") +@patch.object(pd, "read_csv") +def test_get_ephys_features_with_api(read_csv, to_csv, cache_fixture, df, @@ -436,7 +435,7 @@ def test_get_ephys_features_with_api(from_csv, _ = ctc.get_ephys_features(dataframe=df) if path_exists: - from_csv.assert_called_once_with(_MOCK_PATH) + read_csv.assert_called_once_with(_MOCK_PATH) else: mkd.assert_called_once_with(_MOCK_PATH) assert query_mock.called @@ -445,11 +444,11 @@ def test_get_ephys_features_with_api(from_csv, @pytest.mark.parametrize('path_exists,df', it.product((False, True), (False, True))) -@patch.object(DataFrame, "to_csv") -@patch.object(DataFrame, "from_csv", - return_value=DataFrame([{ 'stuff': 'whatever'}, - { 'stuff': 'nonsense'}])) -def test_get_morphology_features(from_csv, +@patch.object(pd.DataFrame, "to_csv") +@patch.object(pd, "read_csv", + return_value=pd.DataFrame([{ 'stuff': 'whatever'}, + { 'stuff': 'nonsense'}])) +def test_get_morphology_features(read_csv, to_csv, cache_fixture, path_exists, @@ -479,7 +478,7 @@ def test_get_morphology_features(from_csv, if path_exists: if df: - from_csv.assert_called_once_with(_MOCK_PATH) + read_csv.assert_called_once_with(_MOCK_PATH) else: assert True assert not mkd.called @@ -546,11 +545,11 @@ def test_get_ephys_sweeps_with_api(cache_fixture, it.product((False, True), (False, True))) @patch('pandas.DataFrame.merge') -@patch.object(DataFrame, "to_csv") -@patch.object(DataFrame, "from_csv", - return_value=DataFrame([{ 'stuff': 'whatever'}, - { 'stuff': 'nonsense'}])) -def test_get_all_features(from_csv, +@patch.object(pd.DataFrame, "to_csv") +@patch.object(pd, "read_csv", + return_value=pd.DataFrame([{ 'stuff': 'whatever'}, + { 'stuff': 'nonsense'}])) +def test_get_all_features(read_csv, to_csv, mock_merge, cache_fixture, @@ -578,7 +577,7 @@ def test_get_all_features(from_csv, require_reconstruction=require_reconstruction) if path_exists: - assert from_csv.called + assert read_csv.called else: assert query_mock.called diff --git a/doc_template/data_api_client.rst b/doc_template/data_api_client.rst index af0f45acc..2e9a8eeb0 100644 --- a/doc_template/data_api_client.rst +++ b/doc_template/data_api_client.rst @@ -166,7 +166,7 @@ Pandas Dataframes can be written to a CSV file using to_csv and read using load_ 'parent_structure_id', 'acronym']].to_csv('summary_structures.csv', index_label='structure_id') - reread = pd.DataFrame.from_csv('summary_structures.csv') + reread = pd.read_csv('summary_structures.csv') Iteration over a Dataframe of API data can be done in several ways.