Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Single Snapshot on Root with RollBack #278

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

Single Snapshot on Root with RollBack #278

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
1293aae
Compile Successful
May 8, 2015
1b7feae
8June2016
pushparajxa Jun 9, 2016
407e73d
18June16:Merge with master on github
pushparajxa Jun 18, 2016
e1d66c7
RollBackManager
pushparajxa Mar 12, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions hadoop-common-project/hadoop-annotations/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,21 @@
</dependency>
</dependencies>
</profile>
<profile>
<id>jdk1.8</id>
<activation>
<jdk>1.8</jdk>
</activation>
<dependencies>
<dependency>
<groupId>jdk.tools</groupId>
<artifactId>jdk.tools</artifactId>
<version>1.8</version>
<scope>system</scope>
<systemPath>${java.home}/../lib/tools.jar</systemPath>
</dependency>
</dependencies>
</profile>
</profiles>

</project>
17 changes: 17 additions & 0 deletions hadoop-hdfs-project/hadoop-hdfs/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,23 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
</properties>

<dependencies>
<dependency>
<groupId>com.mysql.ndb</groupId>
<artifactId>clusterj-part-key-fix</artifactId>
<!--artifactId>clusterj</artifactId-->
<version>7.4.7</version>
</dependency>
<dependency>
<groupId>com.mysql.ndb</groupId>
<artifactId>clusterj</artifactId>
<version>7.4.4</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.18</version>
</dependency>

<dependency>
<groupId>io.hops</groupId>
<artifactId>hops-leader-election</artifactId>
Expand Down
10 changes: 2 additions & 8 deletions hadoop-hdfs-project/hadoop-hdfs/src/main/java/io/hops/metadata/HdfsStorageFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,14 +98,7 @@
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
import org.apache.hadoop.hdfs.server.blockmanagement.PendingBlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.ReplicaUnderConstruction;
import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INodeAttributes;
import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
import org.apache.hadoop.hdfs.server.namenode.INodeDirectoryWithQuota;
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.server.namenode.INodeFileUnderConstruction;
import org.apache.hadoop.hdfs.server.namenode.INodeSymlink;
import org.apache.hadoop.hdfs.server.namenode.Lease;
import org.apache.hadoop.hdfs.server.namenode.*;

import java.io.File;
import java.io.IOException;
Expand Down Expand Up @@ -216,6 +209,7 @@ private static void initDataAccessWrappers() {
dataAccessAdaptors.put(INodeAttributesDataAccess.class,
new INodeAttributeDALAdaptor((INodeAttributesDataAccess) getDataAccess(
INodeAttributesDataAccess.class)));

}

private static ContextInitializer getContextInitializer() {
Expand Down
162 changes: 153 additions & 9 deletions hadoop-hdfs-project/hadoop-hdfs/src/main/java/io/hops/metadata/HdfsVariables.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,18 +19,18 @@
import io.hops.exception.StorageException;
import io.hops.exception.TransactionContextException;
import io.hops.leaderElection.VarsRegister;
import io.hops.metadata.common.entity.ArrayVariable;
import io.hops.metadata.common.entity.ByteArrayVariable;
import io.hops.metadata.common.entity.IntVariable;
import io.hops.metadata.common.entity.LongVariable;
import io.hops.metadata.common.entity.Variable;
import io.hops.metadata.common.entity.*;
import io.hops.metadata.hdfs.dal.VariableDataAccess;
import io.hops.metadata.hdfs.snapshots.SnapShotConstants;
import io.hops.transaction.handler.HDFSOperationType;
import io.hops.transaction.handler.LightWeightRequestHandler;
import org.apache.commons.digester.substitution.VariableAttributes;
import org.apache.commons.io.output.ByteArrayOutputStream;
import org.apache.hadoop.hdfs.security.token.block.BlockKey;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.hdfs.server.namenode.RemoveSnapshotManager;
import org.apache.hadoop.hdfs.server.namenode.RollBackManager;
import org.apache.hadoop.hdfs.server.namenode.SnapShotManager;

import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
Expand Down Expand Up @@ -252,8 +252,49 @@ public static void setSIdCounter(int sid)
throws StorageException, TransactionContextException {
Variables.updateVariable(new IntVariable(Variable.Finder.SIdCounter, sid));
}
public static long getMaxNNID()throws StorageException, TransactionContextException {
return (Long) Variables.getVariable(Variable.Finder.MaxNNID).getValue();
}

private static Map<Integer, BlockKey> getAllBlockTokenKeys(boolean useKeyId,
public static void setMaxNNID(long val) throws StorageException, TransactionContextException {
Variables.updateVariable(new LongVariable(Variable.Finder.MaxNNID, val));
}

public static String getRollBackStatus() throws StorageException, TransactionContextException {
return (String) Variables.getVariable(Variable.Finder.RollBackStatus).getValue();
}

public static void setRollBackStatus(String status) throws StorageException, TransactionContextException {

Variables.updateVariable(new StringVariable(Variable.Finder.RollBackStatus, status));
}

public static String getRollBackStatus2() throws IOException {
return (String) new LightWeightRequestHandler(HDFSOperationType.GET_ROLLBACK_STATUS) {
@Override
public Object performTask() throws IOException {
VariableDataAccess vd = (VariableDataAccess) HdfsStorageFactory.getDataAccess(VariableDataAccess.class);
HdfsStorageFactory.getConnector().readCommitted();
return ((StringVariable) vd.getVariable(Variable.Finder.RollBackStatus)).getValue();
}
}.handle();

}

public static boolean setRollBackStatus2(final RollBackManager.RollBackStatus rollBackStatus) throws IOException {
return (Boolean) new LightWeightRequestHandler(HDFSOperationType.SET_ROLLBACK_STATUS) {
@Override
public Object performTask() throws IOException {
VariableDataAccess vd = (VariableDataAccess) HdfsStorageFactory.getDataAccess(VariableDataAccess.class);
HdfsStorageFactory.getConnector().writeLock();
vd.setVariable(new StringVariable(Variable.Finder.RollBackStatus, rollBackStatus.toString()));
return true;
}
}.handle();

}

private static Map<Integer, BlockKey> getAllBlockTokenKeys(boolean useKeyId,
boolean leightWeight) throws IOException {
List<Variable> vars = (List<Variable>) (leightWeight ?
getVariableLightWeight(Variable.Finder.BlockTokenKeys).getValue() :
Expand Down Expand Up @@ -307,13 +348,12 @@ public Object performTask() throws IOException {
}
}.handle();
}

public static void registerDefaultValues() {
Variable.registerVariableDefaultValue(Variable.Finder.BlockID,
new LongVariable(0).getBytes());
Variable.registerVariableDefaultValue(Variable.Finder.INodeID,
new IntVariable(2)
.getBytes()); // 1 is taken by the root and zero is parent of the root
new IntVariable(3)
.getBytes()); // 2 is taken by the root and 1 is parent of the root
Variable.registerVariableDefaultValue(Variable.Finder.ReplicationIndex,
new ArrayVariable(Arrays.asList(0, 0, 0, 0, 0)).getBytes());
Variable.registerVariableDefaultValue(Variable.Finder.SIdCounter,
Expand All @@ -328,5 +368,109 @@ public static void registerDefaultValues() {
VarsRegister.registerHdfsDefaultValues();
// This is a workaround that is needed until HA-YARN has its own format command
VarsRegister.registerYarnDefaultValues();
Variable.registerVariableDefaultValue(Variable.Finder.RollBackStatus,new StringVariable(Variable.Finder.RollBackStatus,RollBackManager.RollBackStatus.NOT_STARTED.toString()).getBytes());
Variable.registerVariableDefaultValue(Variable.Finder.RollBackRequestStatus,new IntVariable(Variable.Finder.RollBackRequestStatus, SnapShotConstants.snapShotNotTaken).getBytes());
Variable.registerVariableDefaultValue(Variable.Finder.SnapShotStatus,new StringVariable(Variable.Finder.SnapShotStatus, SnapShotManager.SnapshotStatus.NO_SNAPSHOT.toString()).getBytes());
Variable.registerVariableDefaultValue(Variable.Finder.SnapshotRequestStatus,new StringVariable(Variable.Finder.SnapshotRequestStatus, SnapShotManager.SnapshotRequestStatus.NOT_REQUESTED.toString()).getBytes());
}

public static String getSnapShotStatus() throws IOException{
return (String) new LightWeightRequestHandler(HDFSOperationType.GET_SNAPSHOT_STATUS) {
@Override
public Object performTask() throws IOException {
VariableDataAccess vd = (VariableDataAccess) HdfsStorageFactory.getDataAccess(VariableDataAccess.class);
HdfsStorageFactory.getConnector().readCommitted();
return vd.getVariable(Variable.Finder.SnapShotStatus);
}
}.handle();

}

public static boolean setSnapShotStatus(final SnapShotManager.SnapshotStatus snapShotTakenStatus ) throws IOException {
return (Boolean) new LightWeightRequestHandler(HDFSOperationType.SET_SNAPSHOT_STATUS) {
@Override
public Object performTask() throws IOException {
VariableDataAccess vd = (VariableDataAccess) HdfsStorageFactory.getDataAccess(VariableDataAccess.class);
HdfsStorageFactory.getConnector().writeLock();
vd.setVariable(new StringVariable(Variable.Finder.SnapShotStatus, snapShotTakenStatus.toString()));
return true;
}
}.handle();

}

public static String getSnapShotRequestStatus() throws IOException{
return (String) new LightWeightRequestHandler(HDFSOperationType.GET_SNAPSHOT_REQUEST_STATUS) {
@Override
public Object performTask() throws IOException {
VariableDataAccess vd = (VariableDataAccess) HdfsStorageFactory.getDataAccess(VariableDataAccess.class);
HdfsStorageFactory.getConnector().readCommitted();
return vd.getVariable(Variable.Finder.SnapshotRequestStatus);
}
}.handle();

}

public static boolean setSnapShotRequestStatus(final SnapShotManager.SnapshotRequestStatus snapShotRequestStatus ) throws IOException {
return (Boolean) new LightWeightRequestHandler(HDFSOperationType.SET_SNAPSHOT_REQUEST_STATUS) {
@Override
public Object performTask() throws IOException {
VariableDataAccess vd = (VariableDataAccess) HdfsStorageFactory.getDataAccess(VariableDataAccess.class);
HdfsStorageFactory.getConnector().writeLock();
vd.setVariable(new StringVariable(Variable.Finder.SnapshotRequestStatus, snapShotRequestStatus.toString()));
return true;
}
}.handle();

}
public static String getRollBackRequestStatus() throws IOException{
return (String) new LightWeightRequestHandler(HDFSOperationType.GET_ROLLBACK_REQUEST_STATUS) {
@Override
public Object performTask() throws IOException {
VariableDataAccess vd = (VariableDataAccess) HdfsStorageFactory.getDataAccess(VariableDataAccess.class);
HdfsStorageFactory.getConnector().readCommitted();
return vd.getVariable(Variable.Finder.RollBackRequestStatus);
}
}.handle();

}

public static boolean setRollBackRequestStatus(final RollBackManager.RollBackRequestStatus rollBackRequestStatus ) throws IOException {
return (Boolean) new LightWeightRequestHandler(HDFSOperationType.SET_ROLLBACK_REQUEST_STATUS) {
@Override
public Object performTask() throws IOException {
VariableDataAccess vd = (VariableDataAccess) HdfsStorageFactory.getDataAccess(VariableDataAccess.class);
HdfsStorageFactory.getConnector().writeLock();
vd.setVariable(new StringVariable(Variable.Finder.RollBackRequestStatus, rollBackRequestStatus.toString()));
return true;
}
}.handle();

}

public static String getRemoveSnapshotStatus() throws IOException{
return (String) new LightWeightRequestHandler(HDFSOperationType.GET_REMOVE_SNAPSHOT_STATUS) {
@Override
public Object performTask() throws IOException {
VariableDataAccess vd = (VariableDataAccess) HdfsStorageFactory.getDataAccess(VariableDataAccess.class);
HdfsStorageFactory.getConnector().readCommitted();
return vd.getVariable(Variable.Finder.RemoveSnapshotStatus);
}
}.handle();

}

public static boolean setRemoveSnapshotStatus(final RemoveSnapshotManager.ExecutionStatus removeSnapshotStatus ) throws IOException {
return (Boolean) new LightWeightRequestHandler(HDFSOperationType.SET_REMOVE_SNAPSHOT_STATUS) {
@Override
public Object performTask() throws IOException {
VariableDataAccess vd = (VariableDataAccess) HdfsStorageFactory.getDataAccess(VariableDataAccess.class);
HdfsStorageFactory.getConnector().writeLock();
vd.setVariable(new StringVariable(Variable.Finder.RemoveSnapshotStatus, removeSnapshotStatus.toString()));
return true;
}
}.handle();

}

}
3 changes: 2 additions & 1 deletion ...-hdfs-project/hadoop-hdfs/src/main/java/io/hops/metadata/adaptor/BlockInfoDALAdaptor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ public BlockInfo convertHDFStoDAL(
new BlockInfo(hdfsClass.getBlockId(), hdfsClass.getBlockIndex(),
hdfsClass.getInodeId(), hdfsClass.getNumBytes(),
hdfsClass.getGenerationStamp(),
hdfsClass.getBlockUCState().ordinal(), hdfsClass.getTimestamp());
hdfsClass.getBlockUCState().ordinal(), hdfsClass.getTimestamp(),hdfsClass.getStatus());
if (hdfsClass instanceof BlockInfoUnderConstruction) {
BlockInfoUnderConstruction ucBlock =
(BlockInfoUnderConstruction) hdfsClass;
Expand Down Expand Up @@ -153,6 +153,7 @@ public org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo convertDALtoHDFS(
dalClass.getInodeId());
}

blockInfo.setStatusNoPersistance(dalClass.getStatus());
blockInfo.setINodeIdNoPersistance(dalClass.getInodeId());
blockInfo.setTimestampNoPersistance(dalClass.getTimeStamp());
blockInfo.setBlockIndexNoPersistance(dalClass.getBlockIndex());
Expand Down
4 changes: 2 additions & 2 deletions ...-project/hadoop-hdfs/src/main/java/io/hops/metadata/adaptor/INodeAttributeDALAdaptor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ public INodeAttributes convertHDFStoDAL(
INodeAttributes hia =
new INodeAttributes(attribute.getInodeId(), attribute.getNsQuota(),
attribute.getNsCount(), attribute.getDsQuota(),
attribute.getDiskspace());
attribute.getDiskspace(), attribute.getStatus());
return hia;
} else {
return null;
Expand All @@ -73,7 +73,7 @@ public org.apache.hadoop.hdfs.server.namenode.INodeAttributes convertDALtoHDFS(
org.apache.hadoop.hdfs.server.namenode.INodeAttributes iNodeAttributes =
new org.apache.hadoop.hdfs.server.namenode.INodeAttributes(
hia.getInodeId(), hia.getNsQuota(), hia.getNsCount(),
hia.getDsQuota(), hia.getDiskspace());
hia.getDsQuota(), hia.getDiskspace(),hia.getStatus());
return iNodeAttributes;
} else {
return null;
Expand Down
25 changes: 24 additions & 1 deletion hadoop-hdfs-project/hadoop-hdfs/src/main/java/io/hops/metadata/adaptor/INodeDALAdaptor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,26 @@ public List<org.apache.hadoop.hdfs.server.namenode.INode> indexScanFindInodesByP
.sort(list, org.apache.hadoop.hdfs.server.namenode.INode.Order.ByName);
return list;
}

/**
* After taking snapshot at root, when a pre-existing INode is deleted then it's isDeleted flag is set to 1.In this method we
* will retrieve these children too.
* @param parentId
* @return
* @throws StorageException
*/
@Override
public List<org.apache.hadoop.hdfs.server.namenode.INode> indexScanFindInodesByParentIdIncludeDeletes(int parentId) throws StorageException {
List<org.apache.hadoop.hdfs.server.namenode.INode> list = (List) convertDALtoHDFS(dataAccess.indexScanFindInodesByParentIdIncludeDeletes(parentId));
return list;
}
@Override
public List<ProjectedINode> findInodesByParentIdForSubTreeOpsWithReadLockIncludeDeletes(
int parentId) throws StorageException {
List<ProjectedINode> list =
dataAccess.findInodesByParentIdForSubTreeOpsWithReadLockIncludeDeletes(parentId);
Collections.sort(list);
return list;
}
@Override
public List<ProjectedINode> findInodesForSubtreeOperationsWithWriteLock(
int parentId) throws StorageException {
Expand Down Expand Up @@ -170,6 +189,8 @@ public INode convertHDFStoDAL(
hopINode.setPermission(permissionString.getData());
hopINode.setParentId(inode.getParentId());
hopINode.setId(inode.getId());
hopINode.setStatus(inode.getStatus());
hopINode.setIsDeleted(inode.getIsDeleted());

if (inode.isDirectory()) {
hopINode.setUnderConstruction(false);
Expand Down Expand Up @@ -265,6 +286,8 @@ public org.apache.hadoop.hdfs.server.namenode.INode convertDALtoHDFS(
inode.setParentIdNoPersistance(hopINode.getParentId());
inode.setSubtreeLocked(hopINode.isSubtreeLocked());
inode.setSubtreeLockOwner(hopINode.getSubtreeLockOwner());
inode.setStatusNoPersistance(hopINode.getStatus());
inode.setIsDeletedNoPersistance(hopINode.getIsDeleted());
}
return inode;
}
Expand Down
2 changes: 1 addition & 1 deletion ...project/hadoop-hdfs/src/main/java/io/hops/transaction/context/INodeAttributesContext.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ private void updateAttributes(INodeCandidatePrimaryKey trg_param,

private INodeAttributes clone(INodeAttributes src, int inodeId) {
return new INodeAttributes(inodeId, src.getNsQuota(), src.getNsCount(),
src.getDsQuota(), src.getDiskspace());
src.getDsQuota(), src.getDiskspace(),src.getStatus());
}

}
9 changes: 8 additions & 1 deletion ...hdfs-project/hadoop-hdfs/src/main/java/io/hops/transaction/handler/HDFSOperationType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,10 @@ public enum HDFSOperationType implements OperationType {
GET_COMPLETE_BLOCKS_TOTAL,
SET_GEN_STAMP,
//ClusterInfos
GET_ROLLBACK_STATUS,SET_ROLLBACK_STATUS,GET_ROLLBACK_REQUEST_STATUS,SET_ROLLBACK_REQUEST_STATUS,
GET_SNAPSHOT_STATUS,SET_SNAPSHOT_STATUS,GET_SNAPSHOT_REQUEST_STATUS,SET_SNAPSHOT_REQUEST_STATUS,
GET_REMOVE_SNAPSHOT_STATUS,SET_REMOVE_SNAPSHOT_STATUS,
//ClusterInfos
GET_CLUSTER_INFO,
// NNStorage
ADD_STORAGE_INFO,
Expand All @@ -175,6 +179,9 @@ public enum HDFSOperationType implements OperationType {
GET_INODE,
TO_XML_BLOCK_INFO,
TO_XML_CORRUPT_BLOCK_INFO,
//Root Level Single Snapshot
GET_ROOT_BACKUP, ROOT_SUBTREE_LOCK,
DELETE_SINGLE_SNAPSHOT,
// TestNamenodePing
COUNT_LEASE,
// BLockManagerTestUtil
Expand Down Expand Up @@ -299,5 +306,5 @@ public enum HDFSOperationType implements OperationType {
CHECK_FIXED_PARITY,
RECOVER_ENCODING_JOBS,
DELETE_ENCODING_JOBS,
PERSIST_ENCODING_JOB
PERSIST_ENCODING_JOB
}
Loading