diff --git a/README b/README index ad75248..00a910c 100644 --- a/README +++ b/README @@ -37,6 +37,23 @@ Retrieving data is easy! Use SQL-style keyword args to filter data, or lookup an >>> client.get("/resource/nimj-3ivp/193.json", exclude_system_fields=False) {u'geolocation': {u'latitude': u'21.6711', u'needs_recoding': False, u'longitude': u'142.9236'}, u'version': u'C', u':updated_at': 1348778988, u'number_of_stations': u'136', u'region': u'Mariana Islands region', u':created_meta': u'21484', u'occurred_at': u'2012-09-13T11:19:07', u':id': 193, u'source': u'us', u'depth': u'300.70', u'magnitude': u'4.4', u':meta': u'{\n}', u':updated_meta': u'21484', u':position': 193, u'earthquake_id': u'c000cmsq', u':created_at': 1348778988} +Create a dataset + + >>> columns = [{"fieldName": "delegation", "name": "Delegation", "dataTypeName": "text"}, {"fieldName": "members", "name": "Members", "dataTypeName": "number"}] + >>> tags = ["politics", "geography"] + >>> client.create("Delegates", description="List of delegates", columns=columns, row_identifier="delegation", tags=tags, category="Transparency") + {u'id': u'2frc-hyvj', u'name': u'Foo Bar', u'description': u'test dataset', u'publicationStage': u'unpublished', u'columns': [ { u'name': u'Foo', u'dataTypeName': u'text', u'fieldName': u'foo', ... }, { u'name': u'Bar', u'dataTypeName': u'number', u'fieldName': u'bar', ... } ], u'metadata': { u'rowIdentifier': 230641051 }, ... } + +Publish a dataset after creating it (take it out of 'working copy' mode) + + >>> client.publish("/resource/eb9n-hr43.json") + {u'id': u'2frc-hyvj', u'name': u'Foo Bar', u'description': u'test dataset', u'publicationStage': u'unpublished', u'columns': [ { u'name': u'Foo', u'dataTypeName': u'text', u'fieldName': u'foo', ... }, { u'name': u'Bar', u'dataTypeName': u'number', u'fieldName': u'bar', ... } ], u'metadata': { u'rowIdentifier': 230641051 }, ... } + +Set the permissions of a dataset to public or private + + >>> client.set_permission("/resource/eb9n-hr43.json", "public") + + Create a new row in an existing dataset >>> data = [{'Delegation': 'AJU', 'Name': 'Alaska', 'Key': 'AL', 'Entity': 'Juneau'}] diff --git a/README.md b/README.md index a87ca2f..4320f5f 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,23 @@ Retrieving data is easy! Use SQL-style keyword args to filter data, or lookup an >>> client.get("/resource/nimj-3ivp/193.json", exclude_system_fields=False) {u'geolocation': {u'latitude': u'21.6711', u'needs_recoding': False, u'longitude': u'142.9236'}, u'version': u'C', u':updated_at': 1348778988, u'number_of_stations': u'136', u'region': u'Mariana Islands region', u':created_meta': u'21484', u'occurred_at': u'2012-09-13T11:19:07', u':id': 193, u'source': u'us', u'depth': u'300.70', u'magnitude': u'4.4', u':meta': u'{\n}', u':updated_meta': u'21484', u':position': 193, u'earthquake_id': u'c000cmsq', u':created_at': 1348778988} +Create a dataset + + >>> columns = [{"fieldName": "delegation", "name": "Delegation", "dataTypeName": "text"}, {"fieldName": "members", "name": "Members", "dataTypeName": "number"}] + >>> tags = ["politics", "geography"] + >>> client.create("Delegates", description="List of delegates", columns=columns, row_identifier="delegation", tags=tags, category="Transparency") + {u'id': u'2frc-hyvj', u'name': u'Foo Bar', u'description': u'test dataset', u'publicationStage': u'unpublished', u'columns': [ { u'name': u'Foo', u'dataTypeName': u'text', u'fieldName': u'foo', ... }, { u'name': u'Bar', u'dataTypeName': u'number', u'fieldName': u'bar', ... } ], u'metadata': { u'rowIdentifier': 230641051 }, ... } + +Publish a dataset after creating it (take it out of 'working copy' mode) + + >>> client.publish("/resource/eb9n-hr43.json") + {u'id': u'2frc-hyvj', u'name': u'Foo Bar', u'description': u'test dataset', u'publicationStage': u'unpublished', u'columns': [ { u'name': u'Foo', u'dataTypeName': u'text', u'fieldName': u'foo', ... }, { u'name': u'Bar', u'dataTypeName': u'number', u'fieldName': u'bar', ... } ], u'metadata': { u'rowIdentifier': 230641051 }, ... } + +Set the permissions of a dataset to public or private + + >>> client.set_permission("/resource/eb9n-hr43.json", "public") + + Create a new row in an existing dataset >>> data = [{'Delegation': 'AJU', 'Name': 'Alaska', 'Key': 'AL', 'Entity': 'Juneau'}] @@ -77,6 +94,3 @@ Wrap up when you're finished. ## Run tests $ ./runtests tests/ - -## TODO and known issues -- Currently, the client does not support dataset creation. A new import API is under construction, and being tracked [here](https://github.com/socrata/soda-ruby/issues/13). diff --git a/sodapy/__init__.py b/sodapy/__init__.py index 35fb5c1..6a90f3c 100644 --- a/sodapy/__init__.py +++ b/sodapy/__init__.py @@ -62,7 +62,7 @@ def __init__(self, domain, app_token, username=None, password=None, session_adapter["adapter"]) self.uri_prefix = session_adapter["prefix"] else: - self.uri_prefix = "https" + self.uri_prefix = "https://" def authentication_validation(self, username, password, access_token): ''' @@ -76,8 +76,50 @@ def authentication_validation(self, username, password, access_token): " OAuth2.0. Please use only one authentication" " method.") - def create(self, file_object): - raise NotImplementedError() + def create(self, name, **kwargs): + ''' + Create a dataset, including the field types. Optionally, specify args such as: + description : description of the dataset + columns : list of columns (see docs/tests for list structure) + category : must exist in /admin/metadata + tags : array of tag strings + row_identifier : field name of primary key + ''' + public = kwargs.pop("public", False) + published = kwargs.pop("published", False) + + payload = {"name": name} + + if("row_identifier" in kwargs): + payload["metadata"] = { + "rowIdentifier": kwargs.pop("row_identifier", None) + } + + payload.update(kwargs) + payload = _clear_empty_values(payload) + + return self._perform_update("post", "/api/views.json", payload) + + def set_permission(self, resource, permission="private"): + ''' + Set a dataset's permissions to private or public + Options are private, public + ''' + params = { + "method": "setPermission", + "value": "public.read" if permission == "public" else permission + } + resource = resource.rsplit("/", 1)[-1] # just get the dataset id + + return self._perform_request("put", "/api/views/" + resource, params=params) + + def publish(self, resource): + ''' + The create() method creates a dataset in a "working copy" state. + This method publishes it. + ''' + resource = resource.rsplit("/", 1)[-1].split(".")[0] # just get the dataset id + return self._perform_request("post", "/api/views/" + resource + "/publication.json") def get(self, resource, **kwargs): ''' @@ -145,7 +187,7 @@ def replace(self, resource, payload): return self._perform_update("put", resource, payload) def _perform_update(self, method, resource, payload): - if isinstance(payload, list): + if isinstance(payload, list) or isinstance(payload, dict): response = self._perform_request(method, resource, data=json.dumps(payload)) elif isinstance(payload, file): @@ -184,7 +226,7 @@ def _perform_request(self, request_type, resource, **kwargs): raise Exception("Unknown request type. Supported request types are" ": {0}".format(", ".join(request_type_methods))) - uri = "{0}://{1}{2}".format(self.uri_prefix, self.domain, resource) + uri = "{0}{1}{2}".format(self.uri_prefix, self.domain, resource) # set a timeout, just to be safe kwargs["timeout"] = 10 @@ -195,8 +237,9 @@ def _perform_request(self, request_type, resource, **kwargs): if response.status_code not in (200, 202): _raise_for_status(response) - # deletes have no content body, simply return the whole response - if request_type == "delete": + # when responses have no content body (ie. delete, set_permission), + # simply return the whole response + if not response.text: return response # for other request types, return most useful data diff --git a/tests/test_data/create_foobar.txt b/tests/test_data/create_foobar.txt new file mode 100644 index 0000000..ba28411 --- /dev/null +++ b/tests/test_data/create_foobar.txt @@ -0,0 +1,65 @@ +{ + "id": "2frc-hyvj", + "name": "Foo Bar", + "averageRating": 0, + "createdAt": 1448018696, + "description": "test dataset", + "downloadCount": 0, + "newBackend": false, + "numberOfComments": 0, + "oid": 14929734, + "publicationAppendEnabled": false, + "publicationGroup": 5638965, + "publicationStage": "unpublished", + "rowIdentifierColumnId": 230641051, + "rowsUpdatedAt": 1448018697, + "rowsUpdatedBy": "gxfh-uqsf", + "tableId": 5638965, + "totalTimesRated": 0, + "viewCount": 0, + "viewLastModified": 1448018697, + "viewType": "tabular", + "columns": [ + { + "id": 230641050, + "name": "Foo", + "dataTypeName": "text", + "fieldName": "foo", + "position": 1, + "renderTypeName": "text", + "tableColumnId": 32762225, + "format": {} + }, + { + "id": 230641051, + "name": "Bar", + "dataTypeName": "number", + "fieldName": "bar", + "position": 2, + "renderTypeName": "number", + "tableColumnId": 32762226, + "format": {} + } + ], + "metadata": { + "rowIdentifier": 230641051 + }, + "owner": {}, + "query": {}, + "rights": [ + "read", + "write", + "add", + "delete", + "grant", + "add_column", + "remove_column", + "update_column", + "update_view", + "delete_view" + ], + "tableAuthor": {}, + "flags": [ + "default" + ] +} \ No newline at end of file diff --git a/tests/test_data/empty.txt b/tests/test_data/empty.txt new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_soda.py b/tests/test_soda.py index 3f69cd8..6fe92bc 100644 --- a/tests/test_soda.py +++ b/tests/test_soda.py @@ -7,7 +7,7 @@ import json -PREFIX = "mock" +PREFIX = "https://" DOMAIN = "fakedomain.com" PATH = "/songs.json" APPTOKEN = "FakeAppToken" @@ -113,7 +113,7 @@ def test_delete(): client = Socrata(DOMAIN, APPTOKEN, username=USERNAME, password=PASSWORD, session_adapter=mock_adapter) - uri = "{0}://{1}{2}".format(PREFIX, DOMAIN, PATH) + uri = "{0}{1}{2}".format(PREFIX, DOMAIN, PATH) adapter.register_uri("DELETE", uri, status_code=200) response = client.delete(PATH) assert response.status_code == 200 @@ -124,14 +124,91 @@ def test_delete(): assert isinstance(e, requests_mock.exceptions.NoMockAddress) finally: client.close() + +def test_create(): + mock_adapter = {} + mock_adapter["prefix"] = PREFIX + adapter = requests_mock.Adapter() + mock_adapter["adapter"] = adapter + client = Socrata(DOMAIN, APPTOKEN, username=USERNAME, password=PASSWORD, + session_adapter=mock_adapter) + + response_data = "create_foobar.txt" + resource = "/api/views.json" + set_up_mock(adapter, "POST", response_data, 200, resource=resource) + + columns = [ + {"fieldName": "foo", "name": "Foo", "dataTypeName": "text"}, + {"fieldName": "bar", "name": "Bar", "dataTypeName": "number"} + ] + tags = ["foo", "bar"] + response = client.create("Foo Bar", description="test dataset", + columns=columns, tags=tags, row_identifier="bar") + + request = adapter.request_history[0] + request_payload = json.loads(request.text) # can't figure out how to use .json + + # Test request payload + for dataset_key in ["name", "description", "columns", "tags"]: + assert dataset_key in request_payload + + for column_key in ["fieldName", "name", "dataTypeName"]: + assert column_key in request_payload["columns"][0] + + # Test response + assert isinstance(response, dict) + assert len(response.get("id")) == 9 + client.close() + +def test_set_permission(): + mock_adapter = {} + mock_adapter["prefix"] = PREFIX + adapter = requests_mock.Adapter() + mock_adapter["adapter"] = adapter + client = Socrata(DOMAIN, APPTOKEN, username=USERNAME, password=PASSWORD, + session_adapter=mock_adapter) + response_data = "empty.txt" + resource = "/api/views" + PATH + set_up_mock(adapter, "PUT", response_data, 200, resource=resource) + + # Test response + response = client.set_permission(PATH, "public") + assert response.status_code == 200 + + # Test request + request = adapter.request_history[0] + qs = request.url.split("?")[-1] + assert qs == "method=setPermission&value=public.read" + client.close() + +def test_publish(): + mock_adapter = {} + mock_adapter["prefix"] = PREFIX + adapter = requests_mock.Adapter() + mock_adapter["adapter"] = adapter + client = Socrata(DOMAIN, APPTOKEN, username=USERNAME, password=PASSWORD, + session_adapter=mock_adapter) + + response_data = "create_foobar.txt" + resource = "/api/views/songs/publication.json" # publish() removes .json + set_up_mock(adapter, "POST", response_data, 200, resource=resource) + + response = client.publish("/resource/songs.json") # hard-coded so request uri is matched + assert isinstance(response, dict) + assert len(response.get("id")) == 9 + client.close() def set_up_mock(adapter, method, response, response_code, - reason="OK", auth=None): + reason="OK", auth=None, resource=PATH): path = os.path.join(TEST_DATA_PATH, response) with open(path, "rb") as f: - body = json.load(f) - uri = "{0}://{1}{2}".format(PREFIX, DOMAIN, PATH) + try: + body = json.load(f) + except ValueError: + body = None # for case of empty file (test_set_permission) + + uri = "{0}{1}{2}".format(PREFIX, DOMAIN, resource) headers = { "content-type": "application/json; charset=utf-8" }