Commit 33982028 authored by Daniel Gerhardt's avatar Daniel Gerhardt

Continue aborted migrations with help of saved state

Migrations can now be split into smaller steps. This change and
additional stored migration state data allow aborted migrations to be
continued.
parent 0a4c5e47
......@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Objects;
import org.springframework.core.style.ToStringCreator;
import de.thm.arsnova.model.serialization.View;
......@@ -30,6 +31,8 @@ public class MigrationState extends Entity {
public static class Migration {
private String id;
private Date start;
private int step;
private Object state;
public Migration() {
......@@ -50,9 +53,34 @@ public class MigrationState extends Entity {
return start;
}
@JsonView({View.Persistence.class, View.Public.class})
public int getStep() {
return step;
}
@JsonView(View.Persistence.class)
public void setStep(final int step) {
this.step = step;
}
@JsonView({View.Persistence.class, View.Public.class})
public Object getState() {
return state;
}
@JsonView(View.Persistence.class)
public void setState(final Object state) {
this.state = state;
}
@Override
public String toString() {
return "Migration " + id + " started at " + start;
return new ToStringCreator(this)
.append("id", id)
.append("start", start)
.append("step", step)
.append("state", state)
.toString();
}
}
......@@ -134,4 +162,11 @@ public class MigrationState extends Entity {
return Objects.equals(active, that.active)
&& Objects.equals(completed, that.completed);
}
@Override
protected ToStringCreator buildToString() {
return super.buildToString()
.append("active", active)
.append("completed", completed);
}
}
......@@ -132,8 +132,8 @@ public class CouchDbInitializer implements ResourceLoaderAware {
}
protected void migrate(final MigrationState state) {
if (migrationExecutor != null && migrationExecutor.runMigrations(state)) {
connector.update(state);
if (migrationExecutor != null) {
migrationExecutor.runMigrations(state, () -> connector.update(state));
}
}
......@@ -158,8 +158,8 @@ public class CouchDbInitializer implements ResourceLoaderAware {
migrate(state);
statusService.removeMaintenanceReason(this.getClass());
} catch (final DbAccessException e) {
logger.error("Database is invalid.", e);
statusService.putMaintenanceReason(this.getClass(), "Invalid database");
logger.error("Database initialization failed.", e);
statusService.putMaintenanceReason(this.getClass(), "Invalid database state");
}
}
......
......@@ -18,8 +18,12 @@
package de.thm.arsnova.persistence.couchdb.migrations;
import de.thm.arsnova.model.MigrationState;
public interface Migration {
String getId();
void migrate();
int getStepCount();
void migrate(MigrationState.Migration state);
}
......@@ -48,22 +48,43 @@ public class MigrationExecutor {
logger.debug("Initialized {} migration(s).", this.migrations.size());
}
public boolean runMigrations(@NonNull final MigrationState migrationState) {
public boolean runMigrations(@NonNull final MigrationState migrationState, final Runnable stateUpdateHandler) {
final Thread shutdownHook = new Thread(stateUpdateHandler);
Runtime.getRuntime().addShutdownHook(shutdownHook);
final List<Migration> pendingMigrations = migrations.stream()
.filter(m -> !migrationState.getCompleted().contains(m.getId())).collect(Collectors.toList());
boolean stateChange = false;
if (migrationState.getActive() != null) {
throw new IllegalStateException("An migration is already active: " + migrationState.getActive());
}
logger.debug("Pending migrations: " + pendingMigrations.stream()
logger.info("Pending migrations: " + pendingMigrations.stream()
.map(Migration::getId).collect(Collectors.joining()));
for (final Migration migration : pendingMigrations) {
if (migrationState.getActive() != null) {
logger.info("Trying to continue from aborted migration: " + migrationState.getActive());
if (pendingMigrations.isEmpty()
|| migrationState.getActive().getId().equals(pendingMigrations.get(0))) {
throw new IllegalStateException("Migration state does not match next pending migration.");
}
} else {
migrationState.setActive(migration.getId(), new Date());
}
stateChange = true;
migrationState.setActive(migration.getId(), new Date());
migration.migrate();
final int initialStep = migrationState.getActive() != null ? migrationState.getActive().getStep() : 0;
for (int i = initialStep; i < migration.getStepCount(); i++) {
logger.info("Performing migration {} step {}...", migration.getId(), i);
try {
migration.migrate(migrationState.getActive());
} catch (final Exception e) {
logger.info("Current migration state: {}", migrationState);
stateUpdateHandler.run();
throw e;
}
migrationState.getActive().setStep(i + 1);
logger.info("Completed migration {} step {}.", migration.getId(), i);
stateUpdateHandler.run();
}
migrationState.getCompleted().add(migration.getId());
migrationState.setActive(null);
}
Runtime.getRuntime().removeShutdownHook(shutdownHook);
return stateChange;
}
......
......@@ -40,6 +40,7 @@ import de.thm.arsnova.model.Answer;
import de.thm.arsnova.model.Comment;
import de.thm.arsnova.model.Content;
import de.thm.arsnova.model.ContentGroup;
import de.thm.arsnova.model.MigrationState;
import de.thm.arsnova.model.Motd;
import de.thm.arsnova.model.Room;
import de.thm.arsnova.model.UserProfile;
......@@ -83,6 +84,7 @@ public class V2ToV3Migration implements Migration {
private RoomRepository roomRepository;
private ContentRepository contentRepository;
private long referenceTimestamp = System.currentTimeMillis();
private MigrationState.Migration state;
public V2ToV3Migration(
final FromV2Migrator migrator,
......@@ -103,18 +105,44 @@ public class V2ToV3Migration implements Migration {
return ID;
}
public void migrate() {
public int getStepCount() {
return 8;
}
@Override
public void migrate(final MigrationState.Migration state) {
this.state = state;
createV2Index();
migrator.setIgnoreRevision(true);
try {
migrateUsers();
migrateUnregisteredUsers();
migrateRooms();
migrateMotds();
migrateComments();
migrateContents();
migrateContentGroups();
migrateAnswers();
switch (state.getStep()) {
case 0:
migrateUsers();
break;
case 1:
migrateUnregisteredUsers();
break;
case 2:
migrateRooms();
break;
case 3:
migrateMotds();
break;
case 4:
migrateComments();
break;
case 5:
migrateContents();
break;
case 6:
migrateContentGroups();
break;
case 7:
migrateAnswers();
break;
default:
throw new IllegalStateException("Invalid migration step:" + state.getStep() + ".");
}
} catch (final InterruptedException e) {
throw new DbAccessException(e);
}
......@@ -195,7 +223,7 @@ public class V2ToV3Migration implements Migration {
final MangoCouchDbConnector.MangoQuery query = new MangoCouchDbConnector.MangoQuery(queryOptions);
query.setIndexDocument(USER_INDEX);
query.setLimit(LIMIT);
String bookmark = null;
String bookmark = (String) state.getState();
for (int skip = 0;; skip += LIMIT) {
logger.debug("Migration progress: {}, bookmark: {}", skip, bookmark);
......@@ -225,7 +253,9 @@ public class V2ToV3Migration implements Migration {
}
toConnector.executeBulk(profilesV3);
state.setState(bookmark);
}
state.setState(null);
}
private void migrateUnregisteredUsers() throws InterruptedException {
......@@ -253,7 +283,7 @@ public class V2ToV3Migration implements Migration {
query = new MangoCouchDbConnector.MangoQuery(queryOptions);
query.setIndexDocument(LOGGEDIN_INDEX);
query.setLimit(LIMIT);
String bookmark = null;
String bookmark = (String) state.getState();
for (int skip = 0;; skip += LIMIT) {
logger.debug("Migration progress: {}, bookmark: {}", skip, bookmark);
query.setBookmark(bookmark);
......@@ -276,7 +306,9 @@ public class V2ToV3Migration implements Migration {
profilesV3.add(profileV3);
}
toConnector.executeBulk(profilesV3);
state.setState(bookmark);
}
state.setState(null);
}
private void migrateRooms() throws InterruptedException {
......@@ -286,7 +318,7 @@ public class V2ToV3Migration implements Migration {
final MangoCouchDbConnector.MangoQuery query = new MangoCouchDbConnector.MangoQuery(queryOptions);
query.setIndexDocument(SESSION_INDEX);
query.setLimit(LIMIT);
String bookmark = null;
String bookmark = (String) state.getState();
for (int skip = 0;; skip += LIMIT) {
logger.debug("Migration progress: {}, bookmark: {}", skip, bookmark);
......@@ -311,7 +343,9 @@ public class V2ToV3Migration implements Migration {
}
toConnector.executeBulk(roomsV3);
state.setState(bookmark);
}
state.setState(null);
}
private void migrateMotds() throws InterruptedException {
......@@ -325,7 +359,7 @@ public class V2ToV3Migration implements Migration {
final MangoCouchDbConnector.MangoQuery query = new MangoCouchDbConnector.MangoQuery(queryOptions);
query.setIndexDocument(MOTD_INDEX);
query.setLimit(LIMIT);
String bookmark = null;
String bookmark = (String) state.getState();
for (int skip = 0;; skip += LIMIT) {
logger.debug("Migration progress: {}, bookmark: {}", skip, bookmark);
......@@ -354,7 +388,9 @@ public class V2ToV3Migration implements Migration {
}
toConnector.executeBulk(motdsV3);
state.setState(bookmark);
}
state.setState(null);
}
private void migrateComments() throws InterruptedException {
......@@ -364,7 +400,7 @@ public class V2ToV3Migration implements Migration {
final MangoCouchDbConnector.MangoQuery query = new MangoCouchDbConnector.MangoQuery(queryOptions);
query.setIndexDocument(FULL_INDEX_BY_TYPE);
query.setLimit(LIMIT);
String bookmark = null;
String bookmark = (String) state.getState();
for (int skip = 0;; skip += LIMIT) {
logger.debug("Migration progress: {}, bookmark: {}", skip, bookmark);
......@@ -402,7 +438,9 @@ public class V2ToV3Migration implements Migration {
}
toConnector.executeBulk(commentsV3);
state.setState(bookmark);
}
state.setState(null);
}
private void migrateContents() throws InterruptedException {
......@@ -412,7 +450,7 @@ public class V2ToV3Migration implements Migration {
final MangoCouchDbConnector.MangoQuery query = new MangoCouchDbConnector.MangoQuery(queryOptions);
query.setIndexDocument(FULL_INDEX_BY_TYPE);
query.setLimit(LIMIT);
String bookmark = null;
String bookmark = (String) state.getState();
for (int skip = 0;; skip += LIMIT) {
logger.debug("Migration progress: {}, bookmark: {}", skip, bookmark);
......@@ -440,7 +478,9 @@ public class V2ToV3Migration implements Migration {
}
toConnector.executeBulk(contentsV3);
state.setState(bookmark);
}
state.setState(null);
}
private void migrateContentGroups() throws InterruptedException {
......@@ -455,7 +495,7 @@ public class V2ToV3Migration implements Migration {
query.setSort(sort);
query.setIndexDocument(SKILLQUESTION_INDEX);
query.setLimit(LIMIT);
String bookmark = null;
String bookmark = (String) state.getState();
final Map<String, Set<String>> groups = new HashMap<>();
String roomId = "";
......@@ -481,8 +521,10 @@ public class V2ToV3Migration implements Migration {
groups.put(contentV2.getQuestionVariant(), contentIds);
contentIds.add(contentV2.getId());
}
state.setState(bookmark);
}
createContentGroups(roomId, groups);
state.setState(null);
}
private void createContentGroups(final String roomId, final Map<String, Set<String>> groups) {
......@@ -508,7 +550,7 @@ public class V2ToV3Migration implements Migration {
final MangoCouchDbConnector.MangoQuery query = new MangoCouchDbConnector.MangoQuery(queryOptions);
query.setIndexDocument(FULL_INDEX_BY_TYPE);
query.setLimit(LIMIT);
String bookmark = null;
String bookmark = (String) state.getState();
for (int skip = 0;; skip += LIMIT) {
logger.debug("Migration progress: {}, bookmark: {}", skip, bookmark);
......@@ -541,7 +583,9 @@ public class V2ToV3Migration implements Migration {
}
toConnector.executeBulk(answersV3);
state.setState(bookmark);
}
state.setState(null);
}
private HashSet<String> migrateMotdIds(final Set<String> oldIds) throws InterruptedException {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment