From 88ef8bf8b18ef4e85e8ac9a1249f14b452ebbae9 Mon Sep 17 00:00:00 2001
From: Daniel Gerhardt <code@dgerhardt.net>
Date: Mon, 9 Oct 2017 17:56:48 +0200
Subject: [PATCH] Implement database migrations to v3
---
.../thm/arsnova/config/PersistanceConfig.java | 16 ++-
.../thm/arsnova/entities/MigrationState.java | 18 ++-
.../serialization/CouchDbDocumentModule.java | 1 +
.../couchdb/InitializingCouchDbConnector.java | 23 ++++
.../couchdb/migrations/Migration.java | 6 +
.../couchdb/migrations/MigrationExecutor.java | 44 +++++++
.../couchdb/migrations/V2ToV3Migration.java | 107 ++++++++++++++++++
.../support/MangoCouchDbConnector.java | 4 +-
src/main/resources/arsnova.properties.example | 3 +-
src/test/resources/arsnova.properties.example | 3 +-
10 files changed, 216 insertions(+), 9 deletions(-)
create mode 100644 src/main/java/de/thm/arsnova/persistance/couchdb/migrations/Migration.java
create mode 100644 src/main/java/de/thm/arsnova/persistance/couchdb/migrations/MigrationExecutor.java
create mode 100644 src/main/java/de/thm/arsnova/persistance/couchdb/migrations/V2ToV3Migration.java
diff --git a/src/main/java/de/thm/arsnova/config/PersistanceConfig.java b/src/main/java/de/thm/arsnova/config/PersistanceConfig.java
index d278ee91..8add6949 100644
--- a/src/main/java/de/thm/arsnova/config/PersistanceConfig.java
+++ b/src/main/java/de/thm/arsnova/config/PersistanceConfig.java
@@ -3,14 +3,19 @@ package de.thm.arsnova.config;
import de.thm.arsnova.entities.serialization.CouchDbObjectMapperFactory;
import de.thm.arsnova.persistance.*;
import de.thm.arsnova.persistance.couchdb.*;
-import org.ektorp.CouchDbConnector;
+import de.thm.arsnova.persistance.couchdb.support.MangoCouchDbConnector;
import org.ektorp.impl.StdCouchDbInstance;
import org.ektorp.spring.HttpClientFactoryBean;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Primary;
import org.springframework.context.annotation.Profile;
+@ComponentScan({
+ "de.thm.arsnova.persistance.couchdb.migrations"
+})
@Configuration
@Profile("!test")
public class PersistanceConfig {
@@ -19,12 +24,19 @@ public class PersistanceConfig {
@Value("${couchdb.port}") private int couchDbPort;
@Value("${couchdb.username:}") private String couchDbUsername;
@Value("${couchdb.password:}") private String couchDbPassword;
+ @Value("${couchdb.migrate-from:}") private String couchDbMigrateFrom;
@Bean
- public CouchDbConnector couchDbConnector() throws Exception {
+ @Primary
+ public MangoCouchDbConnector couchDbConnector() throws Exception {
return new InitializingCouchDbConnector(couchDbName, couchDbInstance(), new CouchDbObjectMapperFactory());
}
+ @Bean
+ public MangoCouchDbConnector couchDbMigrationConnector() throws Exception {
+ return new MangoCouchDbConnector(couchDbMigrateFrom, couchDbInstance(), new CouchDbObjectMapperFactory());
+ }
+
@Bean
public StdCouchDbInstance couchDbInstance() throws Exception {
return new StdCouchDbInstance(couchDbHttpClientFactory().getObject());
diff --git a/src/main/java/de/thm/arsnova/entities/MigrationState.java b/src/main/java/de/thm/arsnova/entities/MigrationState.java
index 3d1d27b0..34428009 100644
--- a/src/main/java/de/thm/arsnova/entities/MigrationState.java
+++ b/src/main/java/de/thm/arsnova/entities/MigrationState.java
@@ -3,6 +3,7 @@ package de.thm.arsnova.entities;
import com.fasterxml.jackson.annotation.JsonView;
import de.thm.arsnova.entities.serialization.View;
+import java.util.ArrayList;
import java.util.Date;
import java.util.List;
@@ -11,6 +12,10 @@ public class MigrationState implements Entity {
private String id;
private Date start;
+ public Migration() {
+
+ }
+
public Migration(String id, Date start) {
this.id = id;
this.start = start;
@@ -25,25 +30,30 @@ public class MigrationState implements Entity {
public Date getStart() {
return start;
}
+
+ @Override
+ public String toString() {
+ return "Migration " + id + " started at " + start;
+ }
}
- private String id = "MigrationState";
+ public static final String ID = "MigrationState";
private String rev;
private Date creationTimestamp;
private Date updateTimestamp;
private Migration active;
- private List<String> completed;
+ private List<String> completed = new ArrayList<>();
@Override
@JsonView(View.Persistence.class)
public String getId() {
- return id;
+ return ID;
}
@Override
@JsonView(View.Persistence.class)
public void setId(final String id) {
- if (!id.equals(this.id)) {
+ if (!id.equals(this.ID)) {
throw new IllegalArgumentException("ID of this entity must not be changed.");
};
}
diff --git a/src/main/java/de/thm/arsnova/entities/serialization/CouchDbDocumentModule.java b/src/main/java/de/thm/arsnova/entities/serialization/CouchDbDocumentModule.java
index 1579458a..4e71d9d2 100644
--- a/src/main/java/de/thm/arsnova/entities/serialization/CouchDbDocumentModule.java
+++ b/src/main/java/de/thm/arsnova/entities/serialization/CouchDbDocumentModule.java
@@ -28,5 +28,6 @@ public class CouchDbDocumentModule extends SimpleModule {
@Override
public void setupModule(SetupContext context) {
context.setMixInAnnotations(Entity.class, CouchDbDocumentMixIn.class);
+ context.setMixInAnnotations(de.thm.arsnova.entities.migration.v2.Entity.class, CouchDbDocumentMixIn.class);
}
}
diff --git a/src/main/java/de/thm/arsnova/persistance/couchdb/InitializingCouchDbConnector.java b/src/main/java/de/thm/arsnova/persistance/couchdb/InitializingCouchDbConnector.java
index fbe08d2d..acb81dfa 100644
--- a/src/main/java/de/thm/arsnova/persistance/couchdb/InitializingCouchDbConnector.java
+++ b/src/main/java/de/thm/arsnova/persistance/couchdb/InitializingCouchDbConnector.java
@@ -1,6 +1,8 @@
package de.thm.arsnova.persistance.couchdb;
import com.fasterxml.jackson.core.JsonProcessingException;
+import de.thm.arsnova.entities.MigrationState;
+import de.thm.arsnova.persistance.couchdb.migrations.MigrationExecutor;
import de.thm.arsnova.persistance.couchdb.support.MangoCouchDbConnector;
import org.ektorp.CouchDbInstance;
import org.ektorp.DocumentNotFoundException;
@@ -8,6 +10,7 @@ import org.ektorp.impl.ObjectMapperFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
+import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ResourceLoaderAware;
import org.springframework.core.io.ClassPathResource;
import org.springframework.core.io.Resource;
@@ -29,6 +32,7 @@ public class InitializingCouchDbConnector extends MangoCouchDbConnector implemen
private final List<Bindings> docs = new ArrayList<>();
private ResourceLoader resourceLoader;
+ private MigrationExecutor migrationExecutor;
public InitializingCouchDbConnector(final String databaseName, final CouchDbInstance dbInstance) {
super(databaseName, dbInstance);
@@ -73,14 +77,33 @@ public class InitializingCouchDbConnector extends MangoCouchDbConnector implemen
});
}
+ protected void migrate() {
+ MigrationState state;
+ try {
+ state = get(MigrationState.class, MigrationState.ID);
+ } catch (DocumentNotFoundException e) {
+ logger.debug("No migration state found in database.");
+ state = new MigrationState();
+ }
+ if (migrationExecutor != null && migrationExecutor.runMigrations(state)) {
+ update(state);
+ }
+ }
+
@Override
public void afterPropertiesSet() throws Exception {
loadDesignDocFiles();
createDesignDocs();
+ migrate();
}
@Override
public void setResourceLoader(final ResourceLoader resourceLoader) {
this.resourceLoader = resourceLoader;
}
+
+ @Autowired
+ public void setMigrationExecutor(final MigrationExecutor migrationExecutor) {
+ this.migrationExecutor = migrationExecutor;
+ }
}
diff --git a/src/main/java/de/thm/arsnova/persistance/couchdb/migrations/Migration.java b/src/main/java/de/thm/arsnova/persistance/couchdb/migrations/Migration.java
new file mode 100644
index 00000000..ea71ece6
--- /dev/null
+++ b/src/main/java/de/thm/arsnova/persistance/couchdb/migrations/Migration.java
@@ -0,0 +1,6 @@
+package de.thm.arsnova.persistance.couchdb.migrations;
+
+public interface Migration {
+ String getId();
+ void migrate();
+}
diff --git a/src/main/java/de/thm/arsnova/persistance/couchdb/migrations/MigrationExecutor.java b/src/main/java/de/thm/arsnova/persistance/couchdb/migrations/MigrationExecutor.java
new file mode 100644
index 00000000..06d54e53
--- /dev/null
+++ b/src/main/java/de/thm/arsnova/persistance/couchdb/migrations/MigrationExecutor.java
@@ -0,0 +1,44 @@
+package de.thm.arsnova.persistance.couchdb.migrations;
+
+import de.thm.arsnova.entities.MigrationState;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
+
+import java.util.Comparator;
+import java.util.Date;
+import java.util.List;
+import java.util.stream.Collectors;
+
+@Service
+public class MigrationExecutor {
+ private static final Logger logger = LoggerFactory.getLogger(MigrationExecutor.class);
+
+ private List<Migration> migrations;
+
+ public MigrationExecutor(List<Migration> migrations) {
+ this.migrations = migrations.stream()
+ .sorted(Comparator.comparing(Migration::getId)).collect(Collectors.toList());
+ }
+
+ public boolean runMigrations(@NonNull final MigrationState migrationState) {
+ List<Migration> pendingMigrations = migrations.stream()
+ .filter(m -> !migrationState.getCompleted().contains(m.getId())).collect(Collectors.toList());
+ boolean stateChange = false;
+ if (migrationState.getActive() != null) {
+ throw new IllegalStateException("An migration is already active: " + migrationState.getActive());
+ }
+ logger.debug("Pending migrations: " + pendingMigrations.stream()
+ .map(Migration::getId).collect(Collectors.joining()));
+ for (Migration migration : pendingMigrations) {
+ stateChange = true;
+ migrationState.setActive(migration.getId(), new Date());
+ migration.migrate();
+ migrationState.getCompleted().add(migration.getId());
+ migrationState.setActive(null);
+ }
+
+ return stateChange;
+ }
+}
diff --git a/src/main/java/de/thm/arsnova/persistance/couchdb/migrations/V2ToV3Migration.java b/src/main/java/de/thm/arsnova/persistance/couchdb/migrations/V2ToV3Migration.java
new file mode 100644
index 00000000..2ed128af
--- /dev/null
+++ b/src/main/java/de/thm/arsnova/persistance/couchdb/migrations/V2ToV3Migration.java
@@ -0,0 +1,107 @@
+package de.thm.arsnova.persistance.couchdb.migrations;
+
+import de.thm.arsnova.entities.Room;
+import de.thm.arsnova.entities.UserProfile;
+import de.thm.arsnova.entities.migration.FromV2Migrator;
+import de.thm.arsnova.entities.migration.v2.DbUser;
+import de.thm.arsnova.entities.migration.v2.LoggedIn;
+import de.thm.arsnova.entities.migration.v2.MotdList;
+import de.thm.arsnova.persistance.UserRepository;
+import de.thm.arsnova.persistance.couchdb.support.MangoCouchDbConnector;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.stereotype.Service;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+@Service
+public class V2ToV3Migration implements Migration {
+ private static final String ID = "20170914131300";
+ private static final int LIMIT = 200;
+
+ private FromV2Migrator migrator;
+ private MangoCouchDbConnector toConnector;
+ private MangoCouchDbConnector fromConnector;
+ private UserRepository userRepository;
+
+ public V2ToV3Migration(
+ final FromV2Migrator migrator,
+ final MangoCouchDbConnector toConnector,
+ @Qualifier("couchDbMigrationConnector") final MangoCouchDbConnector fromConnector,
+ final UserRepository userRepository) {
+ this.migrator = migrator;
+ this.toConnector = toConnector;
+ this.fromConnector = fromConnector;
+ this.userRepository = userRepository;
+ }
+
+ public String getId() {
+ return ID;
+ }
+
+ public void migrate() {
+ migrateUsers();
+ migrateRooms();
+ }
+
+ private void migrateUsers() {
+ HashMap<String, Object> queryOptions = new HashMap<>();
+ queryOptions.put("type", "userdetails");
+ MangoCouchDbConnector.MangoQuery query = fromConnector.new MangoQuery(queryOptions);
+ query.setLimit(LIMIT);
+
+ for (int skip = 0;; skip += LIMIT) {
+ query.setSkip(skip);
+ List<UserProfile> profilesV3 = new ArrayList<>();
+ List<DbUser> dbUsersV2 = fromConnector.query(query, DbUser.class);
+ if (dbUsersV2.size() == 0) {
+ break;
+ }
+
+ for (DbUser userV2 : dbUsersV2) {
+ HashMap<String, Object> loggedInQueryOptions = new HashMap<>();
+ loggedInQueryOptions.put("type", "logged_in");
+ loggedInQueryOptions.put("user", userV2.getUsername());
+ MangoCouchDbConnector.MangoQuery loggedInQuery = fromConnector.new MangoQuery(loggedInQueryOptions);
+ List<LoggedIn> loggedInList = fromConnector.query(loggedInQuery, LoggedIn.class);
+ LoggedIn loggedIn = loggedInList.size() > 0 ? loggedInList.get(0) : null;
+
+ HashMap<String, Object> motdListQueryOptions = new HashMap<>();
+ motdListQueryOptions.put("type", "motdlist");
+ motdListQueryOptions.put("username", userV2.getUsername());
+ MangoCouchDbConnector.MangoQuery motdlistQuery = fromConnector.new MangoQuery(motdListQueryOptions);
+ List<MotdList> motdListList = fromConnector.query(motdlistQuery, MotdList.class);
+ MotdList motdList = motdListList.size() > 0 ? motdListList.get(0) : null;
+
+ profilesV3.add(migrator.migrate(userV2, loggedIn, motdList));
+ }
+
+ toConnector.executeBulk(profilesV3);
+ }
+ }
+
+ private void migrateRooms() {
+ HashMap<String, Object> queryOptions = new HashMap<>();
+ queryOptions.put("type", "session");
+ MangoCouchDbConnector.MangoQuery query = fromConnector.new MangoQuery(queryOptions);
+ query.setLimit(LIMIT);
+
+ for (int skip = 0;; skip += LIMIT) {
+ query.setSkip(skip);
+ List<Room> roomsV3 = new ArrayList<>();
+ List<de.thm.arsnova.entities.migration.v2.Room> roomsV2 = fromConnector.query(query,
+ de.thm.arsnova.entities.migration.v2.Room.class);
+ if (roomsV2.size() == 0) {
+ break;
+ }
+
+ for (de.thm.arsnova.entities.migration.v2.Room roomV2 : roomsV2) {
+ UserProfile profile = userRepository.findByUsername(roomV2.getCreator());
+ roomsV3.add(migrator.migrate(roomV2, profile));
+ }
+
+ toConnector.executeBulk(roomsV3);
+ }
+ }
+}
diff --git a/src/main/java/de/thm/arsnova/persistance/couchdb/support/MangoCouchDbConnector.java b/src/main/java/de/thm/arsnova/persistance/couchdb/support/MangoCouchDbConnector.java
index 4f4eb619..154b360b 100644
--- a/src/main/java/de/thm/arsnova/persistance/couchdb/support/MangoCouchDbConnector.java
+++ b/src/main/java/de/thm/arsnova/persistance/couchdb/support/MangoCouchDbConnector.java
@@ -209,7 +209,9 @@ public class MangoCouchDbConnector extends StdCouchDbConnector {
} catch (JsonProcessingException e) {
throw new DbAccessException("Could not serialize Mango query.");
}
- List<T> result = restTemplate.post(dbURI.append("_find").toString(), queryString, rh);
+ List<T> result = restTemplate.postUncached(dbURI.append("_find").toString(), queryString, rh);
+ //List<T> result = restTemplate.post(dbURI.append("_find").toString(), new JacksonableEntity(query, objectMapper), rh);
+
logger.debug("Answer from CouchDB Mango query: {}", result);
return result;
diff --git a/src/main/resources/arsnova.properties.example b/src/main/resources/arsnova.properties.example
index 34fa9b0a..b1135e4c 100644
--- a/src/main/resources/arsnova.properties.example
+++ b/src/main/resources/arsnova.properties.example
@@ -38,9 +38,10 @@ security.admin-accounts=
################################################################################
couchdb.host=localhost
couchdb.port=5984
-couchdb.name=arsnova
+couchdb.name=arsnova3
couchdb.username=admin
couchdb.password=
+#couchdb.migrate-from=arsnova
################################################################################
diff --git a/src/test/resources/arsnova.properties.example b/src/test/resources/arsnova.properties.example
index 34fa9b0a..b1135e4c 100644
--- a/src/test/resources/arsnova.properties.example
+++ b/src/test/resources/arsnova.properties.example
@@ -38,9 +38,10 @@ security.admin-accounts=
################################################################################
couchdb.host=localhost
couchdb.port=5984
-couchdb.name=arsnova
+couchdb.name=arsnova3
couchdb.username=admin
couchdb.password=
+#couchdb.migrate-from=arsnova
################################################################################
--
GitLab