Skip to content
This repository has been archived by the owner on Feb 12, 2023. It is now read-only.

Commit

Permalink
Fixed GoogleSpreadsheet trouble with special characters in headers an…
Browse files Browse the repository at this point in the history 
…d added parent-uid to repeat spreadsheets.
  • Loading branch information
@dylanfprice
dylanfprice committed Feb 22, 2011
1 parent d40ec8c commit 4c8abfc
Show file tree
Hide file tree
Showing 3 changed files with 181 additions and 94 deletions.
78 changes: 42 additions & 36 deletions src/org/opendatakit/aggregate/constants/externalservice/SpreadsheetConsts.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,49 +21,55 @@
* @author [email protected]
*
*/
public final class SpreadsheetConsts {

public final class SpreadsheetConsts {

public static final String SPEADSHEET_NAME_LABEL = "Name of Spreadsheet:";
/**
* The Google feed permissions. Of the form '/visibility/projection/'. <BR/>
* See http://code.google.com/apis/spreadsheets/data/3.0/reference.html#
* Visibility and
* http://code.google.com/apis/spreadsheets/data/3.0/reference
* .html#Projection for more info.
*/
/**
* The Google feed permissions. Of the form '/visibility/projection/'. <BR/>
* See http://code.google.com/apis/spreadsheets/data/3.0/reference.html#
* Visibility and http://code.google.com/apis/spreadsheets/data/3.0/reference
* .html#Projection for more info.
*/
public static final String FEED_PERMISSIONS = "/private/full/";
public static final String SPREADSHEETS_SCOPE = "https://spreadsheets.google.com/feeds/";
/**
* The spreadsheets feed prefix. <BR/>
* Spreadsheets metafeed url: SPREADSHEETS_FEED <BR/>
* Specific spreadsheet url: SPREADSHEETS_FEED + spreadsheetKey <BR/>
* See http://code.google.com/apis/spreadsheets/data/3.0/reference.html#Feeds
* for more info.
*/
public static final String SPREADSHEETS_FEED = SPREADSHEETS_SCOPE + "spreadsheets" + FEED_PERMISSIONS;
/**
* The spreadsheets feed prefix. <BR/>
* Spreadsheets metafeed url: SPREADSHEETS_FEED <BR/>
* Specific spreadsheet url: SPREADSHEETS_FEED + spreadsheetKey <BR/>
* See http://code.google.com/apis/spreadsheets/data/3.0/reference.html#Feeds
* for more info.
*/
public static final String SPREADSHEETS_FEED = SPREADSHEETS_SCOPE + "spreadsheets"
+ FEED_PERMISSIONS;
/**
* The worksheets feed prefix.<BR/>
* Worksheets metafeed url: WORKSHEETS_FEED + spreadsheetKey + FEED_PERMISSIONS <BR/>
* Specific worksheet url: WORKSHEETS_FEED + spreadsheetKey + FEED_PERMISSIONS + worksheetId <BR/>
* Worksheets metafeed url: WORKSHEETS_FEED + spreadsheetKey +
* FEED_PERMISSIONS <BR/>
* Specific worksheet url: WORKSHEETS_FEED + spreadsheetKey + FEED_PERMISSIONS
* + worksheetId <BR/>
* See http://code.google.com/apis/spreadsheets/data/3.0/reference.html#Feeds
* for more info.
*/
public static final String WORKSHEETS_FEED = SPREADSHEETS_SCOPE + "worksheets/";
public static final String DOCS_SCOPE = "http://docs.google.com/feeds/";
public static final String DOC_FEED = DOCS_SCOPE + "default" + FEED_PERMISSIONS;
public static final String AUTHORIZE_SPREADSHEET_CREATION = "Authorize Doc Service for Spreadsheet Creation";
public static final String GOOGLE_DOC_EXPLANATION = "Create Google Doc Spreadsheet for Form: ";
public static final String DOCS_PRE_KEY = "spreadsheet%3A";

/**
* The name of the property that specifies the session key for the doc and spreadsheets services
*/
*/
public static final String WORKSHEETS_FEED = SPREADSHEETS_SCOPE + "worksheets/";
public static final String DOCS_SCOPE = "http://docs.google.com/feeds/";
public static final String DOC_FEED = DOCS_SCOPE + "default" + FEED_PERMISSIONS;
public static final String AUTHORIZE_SPREADSHEET_CREATION = "Authorize Doc Service for Spreadsheet Creation";
public static final String GOOGLE_DOC_EXPLANATION = "Create Google Doc Spreadsheet for Form: ";
public static final String DOCS_PRE_KEY = "spreadsheet%3A";

/**
* The name of the property that specifies the session key for the doc and
* spreadsheets services
*/
public static final String OAUTH_TOKEN = "oauthToken";
public static final String OAUTH_TOKEN_SECRET = "oauthTokenSecret";

public static final String OAUTH_TOKEN_SECRET = "oauthTokenSecret";

public static final int WORKSHEET_CREATION_DELAY = 15000;
public static final String COMPLETED_AUTH = "Completed Authorization <br>";

public static final String UNSUPPORTED_CHAR_CLASS = "[*:]";
public static final String REPLACEMENT_CHAR = "-";
}

/**
* Characters which cause problems in Google Spreadsheet headers. Removed in
* the GoogleSpreadsheetHeaderFormatter.
*/
public static final String UNSAFE_CHAR_CLASS = "[\\*\\:\\-\\_]";
}
129 changes: 71 additions & 58 deletions src/org/opendatakit/aggregate/externalservice/GoogleSpreadsheet.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
import org.opendatakit.aggregate.form.Form;
import org.opendatakit.aggregate.format.Row;
import org.opendatakit.aggregate.format.element.LinkElementFormatter;
import org.opendatakit.aggregate.format.header.BasicHeaderFormatter;
import org.opendatakit.aggregate.format.header.GoogleSpreadsheetHeaderFormatter;
import org.opendatakit.aggregate.submission.Submission;
import org.opendatakit.aggregate.submission.SubmissionSet;
import org.opendatakit.aggregate.submission.SubmissionValue;
Expand Down Expand Up @@ -87,8 +87,8 @@ public class GoogleSpreadsheet extends AbstractExternalService implements Extern
private final SpreadsheetService spreadsheetService;

private GoogleSpreadsheet(Form form, CallingContext cc) {
super(form, new LinkElementFormatter(cc.getServerURL(), true, true, true), new BasicHeaderFormatter(true, true,
true), cc);
super(form, new LinkElementFormatter(cc.getServerURL(), true, true, true),
new GoogleSpreadsheetHeaderFormatter(true, true, true), cc);
spreadsheetService = new SpreadsheetService(ServletConsts.APPLICATION_NAME);
// TODO: REMOVE after bug is fixed
// http://code.google.com/p/gdata-java-client/issues/detail?id=103
Expand Down Expand Up @@ -121,15 +121,15 @@ public GoogleSpreadsheet(FormServiceCursor fsc, CallingContext cc)
}

public GoogleSpreadsheet(Form form, String name, String spreadKey, OAuthToken authToken,
ExternalServiceOption externalServiceOption, CallingContext cc)
throws ODKDatastoreException {
ExternalServiceOption externalServiceOption, CallingContext cc) throws ODKDatastoreException {
this(form, cc);
objectEntity = cc.getDatastore().createEntityUsingRelation(GoogleSpreadsheetParameterTable
.assertRelation(cc), cc.getCurrentUser());
objectEntity = cc.getDatastore().createEntityUsingRelation(
GoogleSpreadsheetParameterTable.assertRelation(cc), cc.getCurrentUser());
fsc = FormServiceCursor.createFormServiceCursor(form, ExternalServiceType.GOOGLE_SPREADSHEET,
objectEntity, cc);
fsc.setExternalServiceOption(externalServiceOption);
fsc.setIsExternalServicePrepared(false); // need to perform worksheet creation...
fsc.setIsExternalServicePrepared(false); // need to perform worksheet
// creation...
fsc.setOperationalStatus(OperationalStatus.ACTIVE);
fsc.setEstablishmentDateTime(new Date());
fsc.setUploadCompleted(false);
Expand All @@ -148,23 +148,24 @@ public GoogleSpreadsheet(Form form, String name, String spreadKey, OAuthToken au

@Override
public void abandon() throws ODKDatastoreException {
if ( fsc.getOperationalStatus() != OperationalStatus.COMPLETED ) {
fsc.setOperationalStatus(OperationalStatus.ABANDONED);
persist();
}
if (fsc.getOperationalStatus() != OperationalStatus.COMPLETED) {
fsc.setOperationalStatus(OperationalStatus.ABANDONED);
persist();
}
}

public void persist() throws ODKEntityPersistException {
Datastore ds = cc.getDatastore();
User user = cc.getCurrentUser();
Datastore ds = cc.getDatastore();
User user = cc.getCurrentUser();
ds.putEntities(repeatElementTableIds, user);
ds.putEntity(objectEntity, user);
ds.putEntity(fsc, user);
}

public void delete() throws ODKDatastoreException {
// remove spreadsheet permission as no longer needed
// TODO: test that the revoke REALLY works, can be easy to miss since we ignore exception
// TODO: test that the revoke REALLY works, can be easy to miss since we
// ignore exception
try {
OAuthToken token = getAuthToken();
GoogleOAuthParameters oauthParameters = new GoogleOAuthParameters();
Expand All @@ -184,8 +185,8 @@ public void delete() throws ODKDatastoreException {
for (GoogleSpreadsheetRepeatParameterTable repeat : repeatElementTableIds) {
keys.add(new EntityKey(repeat, repeat.getUri()));
}
Datastore ds = cc.getDatastore();
User user = cc.getCurrentUser();
Datastore ds = cc.getDatastore();
User user = cc.getCurrentUser();
ds.deleteEntities(keys, user);
ds.deleteEntity(new EntityKey(objectEntity, objectEntity.getUri()), user);
ds.deleteEntity(new EntityKey(fsc, fsc.getUri()), user);
Expand Down Expand Up @@ -244,19 +245,17 @@ public void generateWorksheets() throws ODKDatastoreException, IOException, Serv
.assertRelation(cc);

// create repeat worksheets
Datastore ds = cc.getDatastore();
User user = cc.getCurrentUser();
List<FormElementModel> repeatGroupElements = form.getRepeatGroupsInModel();
for (FormElementModel repeatGroupElement : repeatGroupElements) {
Datastore ds = cc.getDatastore();
User user = cc.getCurrentUser();
for (FormElementModel repeatGroupElement : form.getRepeatGroupsInModel()) {
// create the worksheet
headers = headerFormatter.generateHeaders(form, repeatGroupElement, null);
WorksheetEntry repeatWorksheet = executeCreateWorksheet(entry, repeatGroupElement
.getElementName(), headers);

// add the worksheet id to the repeat element table -- NOTE: the added
// entry is not actually persisted here
GoogleSpreadsheetRepeatParameterTable t = ds.createEntityUsingRelation(repeatPrototype,
user);
GoogleSpreadsheetRepeatParameterTable t = ds.createEntityUsingRelation(repeatPrototype, user);
t.setUriGoogleSpreadsheet(objectEntity.getUri());
t.setFormElementKey(repeatGroupElement.constructFormElementKey(form));
t.setWorksheetId(extractWorksheetId(repeatWorksheet));
Expand All @@ -265,7 +264,8 @@ public void generateWorksheets() throws ODKDatastoreException, IOException, Serv

// persist the changes we have made to the repeat element table (changes
// from calling executeCreateWorksheet)
fsc.setIsExternalServicePrepared(true); // we have completed worksheet creation...
fsc.setIsExternalServicePrepared(true); // we have completed worksheet
// creation...
persist();
}

Expand Down Expand Up @@ -303,7 +303,6 @@ private WorksheetEntry executeCreateWorksheet(
int index = 0;
for (CellEntry cell : cells) {
String header = headers.get(index);
header = removeUnsafeCharsFromHeader(header);
cell.changeInputValueLocal(header);
BatchUtils.setBatchId(cell, Integer.toString(index));
BatchUtils.setBatchOperationType(cell, BatchOperationType.UPDATE);
Expand All @@ -329,21 +328,20 @@ private WorksheetEntry executeCreateWorksheet(

@Override
public void insertData(Submission submission) throws ODKExternalServiceException {
if (getReady())
{
if (getReady()) {
try {
// upload base submission values
List<String> headers = headerFormatter.generateHeaders(form, form.getTopLevelGroupElement(),
null);
List<String> headers = headerFormatter.generateHeaders(form,
form.getTopLevelGroupElement(), null);
WorksheetEntry topLevelWorksheet = getWorksheet(objectEntity.getTopLevelWorksheetId());
executeInsertData(submission, headers, topLevelWorksheet);

// upload repeat values
for (GoogleSpreadsheetRepeatParameterTable tableId : repeatElementTableIds) {
FormElementKey elementKey = tableId.getFormElementKey();
FormElementModel element = FormElementModel.retrieveFormElementModel(form, elementKey);
headers = headerFormatter.generateHeaders(form, element, null);

List<SubmissionValue> values = submission.findElementValue(element);
for (SubmissionValue value : values) {
if (value instanceof RepeatSubmissionType) {
Expand All @@ -355,7 +353,8 @@ public void insertData(Submission submission) throws ODKExternalServiceException
}
}
} else {
System.out.println("ERROR: How did a non Repeat Submission Type get in the for loop?");
System.out
.println("ERROR: How did a non Repeat Submission Type get in the for loop?");
}
}
}
Expand All @@ -365,28 +364,43 @@ public void insertData(Submission submission) throws ODKExternalServiceException
}
}

private void executeInsertData(SubmissionSet set, List<String> headers, WorksheetEntry worksheet)
throws ODKDatastoreException, IOException, ServiceException {
/**
* Inserts the data in the given submissionSet as a new entry (i.e. a new row)
* in the given worksheet, including only the data specified by headers.
*
* @param submissionSet
* the set of data from a single submission
* @param headers
* a list of headers corresponding to the headers in worksheet. Only
* the data in submissionSet corresponding to these headers will be
* submitted.
* @param worksheet
* the worksheet representing the worksheet in a Google Spreadsheet.
* @throws ODKDatastoreException
* if there was a problem in the datastore
* @throws IOException
* if there was a problem communicating over the internet with the
* Google Spreadsheet
* @throws ServiceException
* if there was a problem with the GData service
*/
private void executeInsertData(SubmissionSet submissionSet, List<String> headers,
WorksheetEntry worksheet) throws ODKDatastoreException, IOException, ServiceException {
ListEntry newEntry = new ListEntry();
CustomElementCollection values = newEntry.getCustomElements();

Row row = set.getFormattedValuesAsRow(null, formatter, false);
Row row = submissionSet.getFormattedValuesAsRow(null, formatter, true);
List<String> formattedValues = row.getFormattedValues();

String rowString = null;
String headerString = null;
for (int colIndex = 0; colIndex < headers.size(); colIndex++) {
String headerString = headers.get(colIndex);
headerString = removeUnsafeCharsFromHeader(headerString);
String rowString = formattedValues.get(colIndex);
headerString = headers.get(colIndex);
rowString = formattedValues.get(colIndex);
values.setValueLocal(headerString, (rowString == null) ? BasicConsts.SPACE : rowString);
}

spreadsheetService.insert(worksheet.getListFeedUrl(), newEntry);
}

// TODO: figure out better way to handle unsafe chars.
private String removeUnsafeCharsFromHeader(String header) {
return header.replaceAll(SpreadsheetConsts.UNSUPPORTED_CHAR_CLASS,
SpreadsheetConsts.REPLACEMENT_CHAR);
spreadsheetService.insert(worksheet.getListFeedUrl(), newEntry);
}

public WorksheetEntry getWorksheet(String worksheetId) throws IOException, ServiceException {
Expand All @@ -398,8 +412,8 @@ public WorksheetEntry getWorksheet(String worksheetId) throws IOException, Servi
}

public static GoogleSpreadsheet createSpreadsheet(Form form, OAuthToken authToken,
String spreadsheetName, ExternalServiceOption externalServiceOption,
CallingContext cc) throws ODKDatastoreException, ODKExternalServiceException {
String spreadsheetName, ExternalServiceOption externalServiceOption, CallingContext cc)
throws ODKDatastoreException, ODKExternalServiceException {

// setup service
DocsService service = new DocsService(ServletConsts.APPLICATION_NAME);
Expand All @@ -425,9 +439,9 @@ public static GoogleSpreadsheet createSpreadsheet(Form form, OAuthToken authToke
} catch (IOException e) {
// try one more time
try {
updatedEntry = service.insert(new URL(SpreadsheetConsts.DOC_FEED), createdEntry);
updatedEntry = service.insert(new URL(SpreadsheetConsts.DOC_FEED), createdEntry);
} catch (Exception e1) {
throw new ODKExternalServiceException(e1);
throw new ODKExternalServiceException(e1);
}
} catch (Exception e) {
throw new ODKExternalServiceException(e);
Expand All @@ -443,11 +457,11 @@ public static GoogleSpreadsheet createSpreadsheet(Form form, OAuthToken authToke
@Override
public void setUploadCompleted() throws ODKEntityPersistException {
fsc.setUploadCompleted(true);
if ( fsc.getExternalServiceOption() == ExternalServiceOption.UPLOAD_ONLY) {
fsc.setOperationalStatus(OperationalStatus.COMPLETED);
if (fsc.getExternalServiceOption() == ExternalServiceOption.UPLOAD_ONLY) {
fsc.setOperationalStatus(OperationalStatus.COMPLETED);
}
Datastore ds = cc.getDatastore();
User user = cc.getCurrentUser();
Datastore ds = cc.getDatastore();
User user = cc.getCurrentUser();
ds.putEntity(fsc, user);
}

Expand All @@ -460,10 +474,9 @@ public boolean equals(Object obj) {
return false;
}
GoogleSpreadsheet other = (GoogleSpreadsheet) obj;
return (objectEntity == null ? (other.objectEntity == null) :
(other.objectEntity != null && objectEntity.equals(other.objectEntity)))
&& (fsc == null ? (other.fsc == null) :
(other.fsc != null && fsc.equals(other.fsc)));
return (objectEntity == null ? (other.objectEntity == null)
: (other.objectEntity != null && objectEntity.equals(other.objectEntity)))
&& (fsc == null ? (other.fsc == null) : (other.fsc != null && fsc.equals(other.fsc)));
}

/**
Expand All @@ -481,7 +494,7 @@ public int hashCode() {

@Override
public String getDescriptiveTargetString() {
return getSpreadsheetName();
return getSpreadsheetName();
}

}
Loading

0 comments on commit 4c8abfc

Please sign in to comment.