Fixed issue with change of the schema while replication is on

engine/src/main/java/com/arcadedb/database/EmbeddedDatabase.java

@@ -217,7 +217,7 @@ private void openInternal() {
         if (fileManager.getFiles().isEmpty())
           schema.create(mode);
         else
-          schema.load(mode);
+          schema.load(mode, true);
 
         if (mode == PaginatedFile.MODE.READ_WRITE)
           checkForRecovery();

engine/src/main/java/com/arcadedb/engine/TransactionManager.java

@@ -272,7 +272,7 @@ public boolean applyChanges(final WALFile.WALTransaction tx) {
       final PaginatedFile file;
 
       if (!database.getFileManager().existsFile(txPage.fileId)) {
-        LogManager.instance().log(this, Level.WARNING, "Error on restoring transaction. Found deleted file %d", null, txPage.fileId);
+        LogManager.instance().log(this, Level.WARNING, "Error on restoring transaction: received operation on deleted file %d", null, txPage.fileId);
         continue;
       }
 

engine/src/main/java/com/arcadedb/exception/SchemaException.java

@@ -15,14 +15,12 @@
  */
 package com.arcadedb.exception;
 
-import java.io.IOException;
-
 public class SchemaException extends ArcadeDBException {
   public SchemaException(final String s) {
     super(s);
   }
 
-  public SchemaException(String s, IOException e) {
+  public SchemaException(String s, Exception e) {
     super(s, e);
   }
 }

engine/src/main/java/com/arcadedb/index/lsm/LSMTreeIndexMutable.java

@@ -48,7 +48,7 @@ public class LSMTreeIndexMutable extends LSMTreeIndexAbstract {
   private             int                   subIndexFileId      = -1;
   private             LSMTreeIndexCompacted subIndex            = null;
   private final       AtomicLong            statsAdjacentSteps  = new AtomicLong();
-  private final       int                   minPagesToScheduleACompaction;
+  private             int                   minPagesToScheduleACompaction;
   private             int                   currentMutablePages = 0;
 
   /**
@@ -78,24 +78,7 @@ protected LSMTreeIndexMutable(final LSMTreeIndex mainIndex, final DatabaseIntern
   protected LSMTreeIndexMutable(final LSMTreeIndex mainIndex, final DatabaseInternal database, final String name, final boolean unique, final String filePath,
       final int id, final PaginatedFile.MODE mode, final int pageSize) throws IOException {
     super(mainIndex, database, name, unique, filePath, id, mode, pageSize);
-
-    final BasePage currentPage = this.database.getTransaction().getPage(new PageId(file.getFileId(), 0), pageSize);
-
-    int pos = INT_SERIALIZED_SIZE + INT_SERIALIZED_SIZE + BYTE_SERIALIZED_SIZE + INT_SERIALIZED_SIZE;
-
-    // TODO: COUNT THE MUTABLE PAGES FROM THE TAIL BACK TO THE HEAD
-    currentMutablePages = 1;
-
-    subIndexFileId = currentPage.readInt(pos);
-
-    pos += INT_SERIALIZED_SIZE;
-
-    final int len = currentPage.readByte(pos++);
-    this.keyTypes = new byte[len];
-    for (int i = 0; i < len; ++i)
-      this.keyTypes[i] = currentPage.readByte(pos++);
-
-    minPagesToScheduleACompaction = database.getConfiguration().getValueAsInteger(GlobalConfiguration.INDEX_COMPACTION_MIN_PAGES_SCHEDULE);
+    onAfterLoad();
   }
 
   @Override
@@ -107,19 +90,38 @@ public void close() {
 
   @Override
   public void onAfterLoad() {
-    if (subIndexFileId > -1) {
-      try {
+    // RELOAD THE PAGE. THIS CAN BE CALLED AT CREATION OF THE OBJECT (CONSTRUCTOR) OR IN A TX WHEN DATABASE STRUCTURE CHANGES
+    try {
+      final BasePage currentPage = this.database.getTransaction().getPage(new PageId(file.getFileId(), 0), pageSize);
+
+      int pos = INT_SERIALIZED_SIZE + INT_SERIALIZED_SIZE + BYTE_SERIALIZED_SIZE + INT_SERIALIZED_SIZE;
+
+      // TODO: COUNT THE MUTABLE PAGES FROM THE TAIL BACK TO THE HEAD
+      currentMutablePages = 1;
+
+      subIndexFileId = currentPage.readInt(pos);
+
+      pos += INT_SERIALIZED_SIZE;
+
+      final int len = currentPage.readByte(pos++);
+      this.keyTypes = new byte[len];
+      for (int i = 0; i < len; ++i)
+        this.keyTypes[i] = currentPage.readByte(pos++);
+
+      minPagesToScheduleACompaction = database.getConfiguration().getValueAsInteger(GlobalConfiguration.INDEX_COMPACTION_MIN_PAGES_SCHEDULE);
+
+      if (subIndexFileId > 0) {
         subIndex = (LSMTreeIndexCompacted) database.getSchema().getFileById(subIndexFileId);
         subIndex.mainIndex = mainIndex;
         subIndex.keyTypes = keyTypes;
 
-      } catch (Exception e) {
-        LogManager.instance().log(this, Level.SEVERE,
-            "Invalid sub-index for index '%s', ignoring it. WARNING: This could lead on using partial indexes. Please recreate the index from scratch (error=%s)",
-            null, name, e.getMessage());
-
-        database.getSchema().dropIndex(name);
       }
+    } catch (Exception e) {
+      LogManager.instance().log(this, Level.SEVERE,
+          "Invalid sub-index for index '%s', ignoring it. WARNING: This could lead on using partial indexes. Please recreate the index from scratch (error=%s)",
+          null, name, e.getMessage());
+
+      database.getSchema().dropIndex(name);
     }
   }
 

engine/src/main/java/com/arcadedb/query/sql/parser/DropPropertyStatement.java

@@ -20,7 +20,6 @@
 import com.arcadedb.database.Database;
 import com.arcadedb.exception.CommandExecutionException;
 import com.arcadedb.index.Index;
-import com.arcadedb.index.IndexInternal;
 import com.arcadedb.index.TypeIndex;
 import com.arcadedb.query.sql.executor.CommandContext;
 import com.arcadedb.query.sql.executor.InternalResultSet;
@@ -63,8 +62,8 @@ public ResultSet executeDDL(CommandContext ctx) {
     if (!indexes.isEmpty()) {
       if (force) {
         for (final Index index : indexes) {
-          ((IndexInternal) index).drop();
-          ResultInternal result = new ResultInternal();
+          database.getSchema().dropIndex(index.getName());
+          final ResultInternal result = new ResultInternal();
           result.setProperty("operation", "cascade drop index");
           result.setProperty("indexName", index.getName());
           rs.add(result);

engine/src/main/java/com/arcadedb/schema/DocumentType.java

@@ -172,7 +172,7 @@ public void dropProperty(final String propertyName) {
   }
 
   public Index createTypeIndex(final EmbeddedSchema.INDEX_TYPE indexType, final boolean unique, final String... propertyNames) {
-    return schema.createTypeIndex(indexType, unique, name, propertyNames);
+    return schema.createTypeIndex(indexType, unique, name, propertyNames, LSMTreeIndexAbstract.DEF_PAGE_SIZE, LSMTreeIndexAbstract.NULL_STRATEGY.SKIP, null);
   }
 
   public Index createTypeIndex(final EmbeddedSchema.INDEX_TYPE indexType, final boolean unique, String[] propertyNames, final int pageSize) {
@@ -227,7 +227,7 @@ public DocumentType addBucket(final Bucket bucket) {
 
   public DocumentType removeBucket(final Bucket bucket) {
     recordFileChanges(() -> {
-      addBucketInternal(bucket);
+      removeBucketInternal(bucket);
       return null;
     });
     return this;
@@ -529,8 +529,11 @@ public void removeIndexInternal(final TypeIndex index) {
 
     for (IndexInternal idx : index.getIndexesOnBuckets()) {
       final List<IndexInternal> list = bucketIndexesByBucket.get(idx.getAssociatedBucketId());
-      if (list != null)
+      if (list != null) {
         list.remove(idx);
+        if (list.isEmpty())
+          bucketIndexesByBucket.remove(idx.getAssociatedBucketId());
+      }
     }
 
     for (DocumentType parent : parentTypes)
@@ -550,12 +553,10 @@ protected void addBucketInternal(final Bucket bucket) {
   }
 
   protected void removeBucketInternal(final Bucket bucket) {
-    for (DocumentType cl : schema.getTypes()) {
-      if (!cl.hasBucket(bucket.getName()))
-        throw new SchemaException(
-            "Cannot remove the bucket '" + bucket.getName() + "' to the type '" + name + "', because the bucket is not associated to the type '" + cl.getName()
-                + "'");
-    }
+    if (!buckets.contains(bucket))
+      throw new SchemaException(
+          "Cannot remove the bucket '" + bucket.getName() + "' to the type '" + name + "', because the bucket is not associated to the type '" + getName()
+              + "'");
 
     buckets.remove(bucket);
   }

engine/src/main/java/com/arcadedb/schema/EmbeddedSchema.java

@@ -122,7 +122,13 @@ public void create(final PaginatedFile.MODE mode) {
     }
   }
 
-  public void load(final PaginatedFile.MODE mode) throws IOException {
+  public void load(final PaginatedFile.MODE mode, final boolean initialize) throws IOException {
+    files.clear();
+    types.clear();
+    bucketMap.clear();
+    indexMap.clear();
+    dictionary = null;
+
     final Collection<PaginatedFile> filesToOpen = database.getFileManager().getFiles();
 
     // REGISTER THE DICTIONARY FIRST
@@ -154,51 +160,13 @@ else if (mainComponent instanceof IndexInternal)
       }
     }
 
-    for (PaginatedComponent f : files)
-      if (f != null)
-        f.onAfterLoad();
+    if (initialize)
+      initComponents();
 
     readConfiguration();
     updateSecurity();
   }
 
-  public void loadChanges() throws IOException {
-    final Collection<PaginatedFile> filesToOpen = database.getFileManager().getFiles();
-
-    final PaginatedFile.MODE mode = database.getMode();
-
-    final List<PaginatedComponent> newFilesLoaded = new ArrayList<>();
-
-    for (PaginatedFile file : filesToOpen) {
-      if (file != null && !Dictionary.DICT_EXT.equals(file.getFileExtension())) {
-        final PaginatedComponent pf = paginatedComponentFactory.createComponent(file, mode);
-
-        if (pf != null) {
-          if (pf.getId() < files.size() && files.get(pf.getId()) != null)
-            // ALREADY LOADED
-            continue;
-
-          final Object mainComponent = pf.getMainComponent();
-
-          if (mainComponent instanceof Bucket)
-            bucketMap.put(pf.getName(), (Bucket) mainComponent);
-          else if (mainComponent instanceof IndexInternal)
-            indexMap.put(pf.getName(), (IndexInternal) mainComponent);
-
-          registerFile(pf);
-
-          newFilesLoaded.add(pf);
-        }
-      }
-    }
-
-    for (PaginatedComponent f : newFilesLoaded)
-      if (f != null)
-        f.onAfterLoad();
-
-    readConfiguration();
-  }
-
   @Override
   public TimeZone getTimeZone() {
     return timeZone;
@@ -413,13 +381,19 @@ public Index[] getIndexes() {
   public void dropIndex(final String indexName) {
     database.checkPermissionsOnDatabase(SecurityDatabaseUser.DATABASE_ACCESS.UPDATE_SCHEMA);
 
-    final IndexInternal index = indexMap.remove(indexName);
-    if (index == null)
-      return;
-
-    index.drop();
+    recordFileChanges(() -> {
+      multipleUpdate = true;
+      try {
+        final IndexInternal index = indexMap.remove(indexName);
+        if (index == null)
+          return null;
 
-    saveConfiguration();
+        index.drop();
+      } catch (Exception e) {
+        throw new SchemaException("Cannot drop the index '" + indexName + "' (error=" + e + ")", e);
+      }
+      return null;
+    });
   }
 
   @Override
@@ -432,18 +406,18 @@ public Index getIndexByName(final String indexName) {
 
   @Override
   public TypeIndex createTypeIndex(final INDEX_TYPE indexType, final boolean unique, final String typeName, final String... propertyNames) {
-    return createTypeIndex(indexType, unique, typeName, propertyNames, LSMTreeIndexAbstract.DEF_PAGE_SIZE, null);
+    return createTypeIndex(indexType, unique, typeName, propertyNames, LSMTreeIndexAbstract.DEF_PAGE_SIZE, LSMTreeIndexAbstract.NULL_STRATEGY.SKIP, null);
   }
 
   @Override
   public TypeIndex createTypeIndex(final INDEX_TYPE indexType, final boolean unique, final String typeName, final String[] propertyNames, final int pageSize) {
-    return createTypeIndex(indexType, unique, typeName, propertyNames, pageSize, LSMTreeIndexAbstract.NULL_STRATEGY.ERROR, null);
+    return createTypeIndex(indexType, unique, typeName, propertyNames, pageSize, LSMTreeIndexAbstract.NULL_STRATEGY.SKIP, null);
   }
 
   @Override
   public TypeIndex createTypeIndex(final INDEX_TYPE indexType, final boolean unique, final String typeName, final String[] propertyNames, final int pageSize,
       final Index.BuildIndexCallback callback) {
-    return createTypeIndex(indexType, unique, typeName, propertyNames, pageSize, LSMTreeIndexAbstract.NULL_STRATEGY.ERROR, callback);
+    return createTypeIndex(indexType, unique, typeName, propertyNames, pageSize, LSMTreeIndexAbstract.NULL_STRATEGY.SKIP, callback);
   }
 
   @Override
@@ -476,7 +450,7 @@ public TypeIndex createTypeIndex(final INDEX_TYPE indexType, final boolean uniqu
     final List<Bucket> buckets = type.getBuckets(true);
     final Index[] indexes = new Index[buckets.size()];
 
-    return recordFileChanges(() -> {
+    recordFileChanges(() -> {
       database.transaction(() -> {
 
         try {
@@ -490,17 +464,17 @@ public TypeIndex createTypeIndex(final INDEX_TYPE indexType, final boolean uniqu
         } catch (IOException e) {
           throw new SchemaException("Cannot create index on type '" + typeName + "' (error=" + e + ")", e);
         }
-
       }, false, 1, null, (error) -> {
         for (int j = 0; j < indexes.length; j++) {
           final IndexInternal indexToRemove = (IndexInternal) indexes[j];
           if (indexToRemove != null)
             indexToRemove.drop();
         }
       });
-
-      return type.getPolymorphicIndexByProperties(propertyNames);
+      return null;
     });
+
+    return type.getPolymorphicIndexByProperties(propertyNames);
   }
 
   @Override
@@ -606,7 +580,6 @@ private Index createBucketIndex(final DocumentType type, final byte[] keyTypes,
       throw new DatabaseMetadataException("Cannot create index '" + indexName + "' on type '" + typeName + "' because it already exists");
 
     return recordFileChanges(() -> {
-
       final IndexInternal index = indexFactory.createIndex(indexType.name(), database, indexName, unique, databasePath + "/" + indexName,
           PaginatedFile.MODE.READ_WRITE, keyTypes, pageSize, nullStrategy, callback);
 
@@ -731,7 +704,7 @@ public void dropType(final String typeName) {
             sub.addParentType(parent);
         }
 
-        final List<Bucket> buckets = type.getBuckets(false);
+        final List<Bucket> buckets = new ArrayList<>(type.getBuckets(false));
         final Set<Integer> bucketIds = new HashSet<>(buckets.size());
         for (Bucket b : buckets)
           bucketIds.add(b.getId());
@@ -741,8 +714,10 @@ public void dropType(final String typeName) {
           dropIndex(m.getName());
 
         // DELETE ALL ASSOCIATED BUCKETS
-        for (Bucket b : buckets)
+        for (Bucket b : buckets) {
+          type.removeBucket(b);
           dropBucket(b.getName());
+        }
 
         if (type instanceof VertexType)
           database.getGraphEngine().dropVertexType(database, (VertexType) type);
@@ -765,6 +740,12 @@ public void dropBucket(final String bucketName) {
     final Bucket bucket = getBucketByName(bucketName);
 
     recordFileChanges(() -> {
+      for (DocumentType type : types.values()) {
+        if (type.buckets.contains(bucket))
+          throw new SchemaException(
+              "Error on dropping bucket '" + bucketName + "' because it is assigned to type '" + type.getName() + "'. Remove the association first");
+      }
+
       database.getPageManager().deleteFile(bucket.getId());
       try {
         database.getFileManager().dropFile(bucket.getId());
@@ -1306,6 +1287,12 @@ public void registerFile(final PaginatedComponent file) {
     files.set(fileId, file);
   }
 
+  public void initComponents() {
+    for (PaginatedComponent f : files)
+      if (f != null)
+        f.onAfterLoad();
+  }
+
   public boolean isDirty() {
     return dirtyConfiguration;
   }

server/src/main/java/com/arcadedb/server/ha/Replica2LeaderNetworkExecutor.java

@@ -435,7 +435,7 @@ private void installDatabase(final Binary buffer, final String db, final Databas
     // RELOAD THE SCHEMA
     database.getSchema().getEmbedded().close();
     DatabaseContext.INSTANCE.init(database);
-    database.getSchema().getEmbedded().load(PaginatedFile.MODE.READ_ONLY);
+    database.getSchema().getEmbedded().load(PaginatedFile.MODE.READ_WRITE, true);
   }
 
   private void installFile(final Binary buffer, final String db, final DatabaseInternal database, final int fileId, final String fileName) throws IOException {