Skip to content

[ZEPPELIN-1426] User aware storage sync #1424

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
khalidhuseynov wants to merge 1 commit into from
Closed

[ZEPPELIN-1426] User aware storage sync #1424

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -469,7 +469,7 @@ public void reloadAllNotes(AuthenticationInfo subject) throws IOException {
if (notebookRepo instanceof NotebookRepoSync) {
NotebookRepoSync mainRepo = (NotebookRepoSync) notebookRepo;
if (mainRepo.getRepoCount() > 1) {
mainRepo.sync();
mainRepo.sync(subject);
}
}

Expand Down
28 changes: 15 additions & 13 deletions zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,8 @@ public NotebookRepoSync(ZeppelinConfiguration conf) {
}
if (getRepoCount() > 1) {
try {
sync(0, 1);
AuthenticationInfo subject = new AuthenticationInfo("anonymous");
sync(0, 1, subject);
} catch (IOException e) {
LOG.warn("Failed to sync with secondary storage on start {}", e);
}
Expand Down Expand Up @@ -175,12 +176,12 @@ public void remove(String noteId, AuthenticationInfo subject) throws IOException
*
* @throws IOException
*/
void sync(int sourceRepoIndex, int destRepoIndex) throws IOException {
void sync(int sourceRepoIndex, int destRepoIndex, AuthenticationInfo subject) throws IOException {
LOG.info("Sync started");
NotebookRepo srcRepo = getRepo(sourceRepoIndex);
NotebookRepo dstRepo = getRepo(destRepoIndex);
List <NoteInfo> srcNotes = srcRepo.list(null);
List <NoteInfo> dstNotes = dstRepo.list(null);
List <NoteInfo> srcNotes = srcRepo.list(subject);
List <NoteInfo> dstNotes = dstRepo.list(subject);

Map<String, List<String>> noteIDs = notesCheckDiff(srcNotes, srcRepo, dstNotes, dstRepo);
List<String> pushNoteIDs = noteIDs.get(pushKey);
Expand All @@ -192,7 +193,7 @@ void sync(int sourceRepoIndex, int destRepoIndex) throws IOException {
for (String id : pushNoteIDs) {
LOG.info("ID : " + id);
}
pushNotes(pushNoteIDs, srcRepo, dstRepo);
pushNotes(subject, pushNoteIDs, srcRepo, dstRepo);
} else {
LOG.info("Nothing to push");
}
Expand All @@ -202,7 +203,7 @@ void sync(int sourceRepoIndex, int destRepoIndex) throws IOException {
for (String id : pullNoteIDs) {
LOG.info("ID : " + id);
}
pushNotes(pullNoteIDs, dstRepo, srcRepo);
pushNotes(subject, pullNoteIDs, dstRepo, srcRepo);
} else {
LOG.info("Nothing to pull");
}
Expand All @@ -212,28 +213,29 @@ void sync(int sourceRepoIndex, int destRepoIndex) throws IOException {
for (String id : delDstNoteIDs) {
LOG.info("ID : " + id);
}
deleteNotes(delDstNoteIDs, dstRepo);
deleteNotes(subject, delDstNoteIDs, dstRepo);
} else {
LOG.info("Nothing to delete from dest");
}

LOG.info("Sync ended");
}

public void sync() throws IOException {
sync(0, 1);
public void sync(AuthenticationInfo subject) throws IOException {
sync(0, 1, subject);
}

private void pushNotes(List<String> ids, NotebookRepo localRepo,
private void pushNotes(AuthenticationInfo subject, List<String> ids, NotebookRepo localRepo,
NotebookRepo remoteRepo) throws IOException {
for (String id : ids) {
remoteRepo.save(localRepo.get(id, null), null);
remoteRepo.save(localRepo.get(id, subject), subject);
}
}

private void deleteNotes(List<String> ids, NotebookRepo repo) throws IOException {
private void deleteNotes(AuthenticationInfo subject, List<String> ids, NotebookRepo repo)
throws IOException {
for (String id : ids) {
repo.remove(id, null);
repo.remove(id, subject);
}
}

Expand Down
2 changes: 1 addition & 1 deletion zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,7 @@ public void testSyncUpdateMain() throws IOException {
assertEquals(0, notebookRepoSync.get(1,
notebookRepoSync.list(1, null).get(0).getId(), null).getParagraphs().size());
/* apply sync */
notebookRepoSync.sync();
notebookRepoSync.sync(null);
/* check whether added to second storage */
assertEquals(1, notebookRepoSync.get(1,
notebookRepoSync.list(1, null).get(0).getId(), null).getParagraphs().size());
Expand Down